added aa_index data for running ml
This commit is contained in:
parent
39ccd6cdf4
commit
05dd9698c4
1 changed files with 226 additions and 49 deletions
235
scripts/ml/ml_data.py
Executable file → Normal file
235
scripts/ml/ml_data.py
Executable file → Normal file
|
|
@ -59,12 +59,21 @@ def setvars(gene,drug):
|
|||
jacc_score_fn = {'jcc': make_scorer(jaccard_score)}
|
||||
|
||||
#%% FOR LATER: Combine ED logo data
|
||||
#%% FOR LATER: active aa site annotations **DONE on 15/05/2022 as part of generating merged_dfs
|
||||
#%% DONE: active aa site annotations **DONE on 15/05/2022 as part of generating merged_dfs
|
||||
###########################################################################
|
||||
rs = {'random_state': 42}
|
||||
njobs = {'n_jobs': 10}
|
||||
homedir = os.path.expanduser("~")
|
||||
|
||||
geneL_basic = ['pnca']
|
||||
geneL_na = ['gid']
|
||||
geneL_na_ppi2 = ['rpob']
|
||||
geneL_ppi2 = ['alr', 'embb', 'katg']
|
||||
|
||||
#num_type = ['int64', 'float64']
|
||||
num_type = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
|
||||
cat_type = ['object', 'bool']
|
||||
|
||||
#==============
|
||||
# directories
|
||||
#==============
|
||||
|
|
@ -75,40 +84,174 @@ def setvars(gene,drug):
|
|||
#=======
|
||||
# input
|
||||
#=======
|
||||
|
||||
#---------
|
||||
# File 1
|
||||
#---------
|
||||
infile_ml1 = outdir + gene.lower() + '_merged_df3.csv'
|
||||
#infile_ml2 = outdir + gene.lower() + '_merged_df2.csv'
|
||||
|
||||
my_df = pd.read_csv(infile_ml1, index_col = 0)
|
||||
my_df.dtypes
|
||||
my_df_cols = my_df.columns
|
||||
my_features_df = pd.read_csv(infile_ml1, index_col = 0)
|
||||
my_features_df = my_features_df .reset_index(drop = True)
|
||||
my_features_df.index
|
||||
|
||||
geneL_basic = ['pnca']
|
||||
geneL_na = ['gid']
|
||||
geneL_na_ppi2 = ['rpob']
|
||||
geneL_ppi2 = ['alr', 'embb', 'katg']
|
||||
#%% get cols
|
||||
mycols = my_df.columns
|
||||
my_features_df.dtypes
|
||||
mycols = my_features_df.columns
|
||||
|
||||
# # change from numberic to
|
||||
# num_type = ['int64', 'float64']
|
||||
# cat_type = ['object', 'bool']
|
||||
#---------
|
||||
# File 2
|
||||
#---------
|
||||
infile_aaindex = outdir + 'aa_index/' + gene.lower() + '_aa.csv'
|
||||
aaindex_df = pd.read_csv(infile_aaindex, index_col = 0)
|
||||
aaindex_df.dtypes
|
||||
|
||||
# if my_df['active_aa_pos'].dtype in num_type:
|
||||
# my_df['active_aa_pos'] = my_df['active_aa_pos'].astype(object)
|
||||
# my_df['active_aa_pos'].dtype
|
||||
#-----------
|
||||
# check for non-numerical columns
|
||||
#-----------
|
||||
if any(aaindex_df.dtypes==object):
|
||||
print('\naaindex_df contains non-numerical data')
|
||||
|
||||
# FIXME: if this is not structural, remove from source..
|
||||
aaindex_df_object = aaindex_df.select_dtypes(include = cat_type)
|
||||
print('\nTotal no. of non-numerial columns:', len(aaindex_df_object.columns))
|
||||
|
||||
expected_aa_ncols = len(aaindex_df.columns) - len(aaindex_df_object.columns)
|
||||
|
||||
#-----------
|
||||
# Extract numerical data only
|
||||
#-----------
|
||||
print('\nSelecting numerical data only')
|
||||
aaindex_df = aaindex_df.select_dtypes(include = num_type)
|
||||
|
||||
#---------------------------
|
||||
# aaindex: sanity check 1
|
||||
#---------------------------
|
||||
if len(aaindex_df.columns) == expected_aa_ncols:
|
||||
print('\nPASS: successfully selected numerical columns only for aaindex_df')
|
||||
else:
|
||||
print('\nFAIL: Numbers mismatch'
|
||||
, '\nExpected ncols:', expected_aa_ncols
|
||||
, '\nGot:', len(aaindex_df.columns))
|
||||
|
||||
#---------------
|
||||
# check for NA
|
||||
#---------------
|
||||
print('\nNow checking for NA in the remaining aaindex_cols')
|
||||
c1 = aaindex_df.isna().sum()
|
||||
c2 = c1.sort_values(ascending=False)
|
||||
print('\nCounting aaindex_df cols with NA'
|
||||
, '\nncols with NA:', sum(c2>0), 'columns'
|
||||
, '\nDropping these...'
|
||||
, '\nOriginal ncols:', len(aaindex_df.columns)
|
||||
)
|
||||
aa_df = aaindex_df.dropna(axis=1)
|
||||
|
||||
print('\nRevised df ncols:', len(aa_df.columns))
|
||||
|
||||
c3 = aa_df.isna().sum()
|
||||
c4 = c3.sort_values(ascending=False)
|
||||
|
||||
print('\nChecking NA in revised df...')
|
||||
|
||||
if sum(c4>0):
|
||||
sys.exit('\nFAIL: aaindex_df still contains cols with NA, please check and drop these before proceeding...')
|
||||
else:
|
||||
print('\nPASS: cols with NA successfully dropped from aaindex_df'
|
||||
, '\nProceeding with combining aa_df with other features_df')
|
||||
|
||||
#---------------------------
|
||||
# aaindex: sanity check 2
|
||||
#---------------------------
|
||||
expected_aa_ncols2 = len(aaindex_df.columns) - sum(c2>0)
|
||||
if len(aa_df.columns) == expected_aa_ncols2:
|
||||
print('\nPASS: ncols match'
|
||||
, '\nExpected ncols:', expected_aa_ncols2
|
||||
, '\nGot:', len(aa_df.columns))
|
||||
else:
|
||||
print('\nFAIL: Numbers mismatch'
|
||||
, '\nExpected ncols:', expected_aa_ncols2
|
||||
, '\nGot:', len(aa_df.columns))
|
||||
|
||||
# Important: need this to identify aaindex cols
|
||||
aa_df_cols = aa_df.columns
|
||||
print('\nTotal no. of columns in clean aa_df:', len(aa_df_cols))
|
||||
|
||||
###############################################################################
|
||||
#%% Combining my_features_df and aaindex_df
|
||||
#===========================
|
||||
# Merge my_df + aaindex_df
|
||||
#===========================
|
||||
|
||||
if aa_df.columns[aa_df.columns.isin(my_features_df.columns)] == my_features_df.columns[my_features_df.columns.isin(aa_df.columns)]:
|
||||
print('\nMerging on column: mutationinformation')
|
||||
|
||||
if len(my_features_df) == len(aa_df):
|
||||
expected_nrows = len(my_features_df)
|
||||
print('\nProceeding to merge, expected nrows in merged_df:', expected_nrows)
|
||||
else:
|
||||
sys.exit('\nNrows mismatch, cannot merge. Please check'
|
||||
, '\nnrows my_df:', len(my_features_df)
|
||||
, '\nnrows aa_df:', len(aa_df))
|
||||
|
||||
#-----------------
|
||||
# Reset index: mutationinformation
|
||||
# Very important for merging
|
||||
#-----------------
|
||||
aa_df = aa_df.reset_index()
|
||||
|
||||
expected_ncols = len(my_features_df.columns) + len(aa_df.columns) - 1 # for the no. of merging col
|
||||
|
||||
#-----------------
|
||||
# Merge: my_features_df + aa_df
|
||||
#-----------------
|
||||
merged_df = pd.merge(my_features_df
|
||||
, aa_df
|
||||
, on = 'mutationinformation')
|
||||
|
||||
#---------------------------
|
||||
# aaindex: sanity check 3
|
||||
#---------------------------
|
||||
if len(merged_df.columns) == expected_ncols:
|
||||
print('\nPASS: my_features_df and aa_df successfully combined'
|
||||
, '\nnrows:', len(merged_df)
|
||||
, '\nncols:', len(merged_df.columns))
|
||||
else:
|
||||
sys.exit('\nFAIL: could not combine my_features_df and aa_df'
|
||||
, '\nCheck dims and merging cols!')
|
||||
|
||||
#--------
|
||||
# Reassign so downstream code doesn't need to change
|
||||
#--------
|
||||
my_df = merged_df.copy()
|
||||
|
||||
#%% Data: my_df
|
||||
# Check if non structural pos have crept in
|
||||
# IDEALLY remove from source! But for rpoB do it here
|
||||
# Drop NA where numerical cols have them
|
||||
if gene.lower() in geneL_na_ppi2:
|
||||
#D1148 get rid of
|
||||
na_index = my_df['mutationinformation'].index[my_df['mcsm_na_affinity'].apply(np.isnan)]
|
||||
my_df = my_df.drop(index=na_index)
|
||||
|
||||
# FIXME: either impute or remove!
|
||||
# for embb (L114M, F115L, V123L, V125I, V131M) delete for now
|
||||
if gene.lower() in ['embb']:
|
||||
na_index = my_df['mutationinformation'].index[my_df['ligand_distance'].apply(np.isnan)]
|
||||
my_df = my_df.drop(index=na_index)# RERUN embb with the 5 values now present
|
||||
# FIXED: complete data for all muts inc L114M, F115L, V123L, V125I, V131M
|
||||
# if gene.lower() in ['embb']:
|
||||
# na_index = my_df['mutationinformation'].index[my_df['ligand_distance'].apply(np.isnan)]
|
||||
# my_df = my_df.drop(index=na_index)
|
||||
|
||||
# # Sanity check for non-structural positions
|
||||
# print('\nChecking for non-structural postions')
|
||||
# na_index = my_df['mutationinformation'].index[my_df['ligand_distance'].apply(np.isnan)]
|
||||
# if len(na_index) > 0:
|
||||
# print('\nNon-structural positions detected for gene:', gene.lower()
|
||||
# , '\nTotal number of these detected:', len(na_index)
|
||||
# , '\These are at index:', na_index
|
||||
# , '\nOriginal nrows:', len(my_df)
|
||||
# , '\nDropping these...')
|
||||
# my_df = my_df.drop(index=na_index)
|
||||
# print('\nRevised nrows:', len(my_df))
|
||||
# else:
|
||||
# print('\nNo non-structural positions detected for gene:', gene.lower()
|
||||
# , '\nnrows:', len(my_df))
|
||||
|
||||
|
||||
###########################################################################
|
||||
#%% Add lineage calculation columns
|
||||
|
|
@ -119,6 +262,12 @@ def setvars(gene,drug):
|
|||
my_df['lineage_proportion'] = my_df['lineage_count_unique']/my_df['lineage_count_all']
|
||||
my_df['dist_lineage_proportion'] = my_df['lineage_count_unique']/total_mtblineage_uc
|
||||
###########################################################################
|
||||
#%% Active site annotation column
|
||||
# change from numberic to categorical
|
||||
|
||||
if my_df['active_site'].dtype in num_type:
|
||||
my_df['active_site'] = my_df['active_site'].astype(object)
|
||||
my_df['active_site'].dtype
|
||||
#%% AA property change
|
||||
#--------------------
|
||||
# Water prop change
|
||||
|
|
@ -213,6 +362,9 @@ def setvars(gene,drug):
|
|||
my_df['aa_prop_change'].dtype
|
||||
|
||||
#%% IMPUTE values for OR [check script for exploration: UQ_or_imputer]
|
||||
#--------------------
|
||||
# Impute OR values
|
||||
#--------------------
|
||||
#or_cols = ['or_mychisq', 'log10_or_mychisq', 'or_fisher']
|
||||
sel_cols = ['mutationinformation', 'or_mychisq', 'log10_or_mychisq']
|
||||
or_cols = ['or_mychisq', 'log10_or_mychisq']
|
||||
|
|
@ -223,7 +375,7 @@ def setvars(gene,drug):
|
|||
my_dfI = pd.DataFrame(index = my_df['mutationinformation'] )
|
||||
|
||||
|
||||
my_dfI = pd.DataFrame(KNN(n_neighbors= 5, weights="uniform").fit_transform(my_df[or_cols])
|
||||
my_dfI = pd.DataFrame(KNN(n_neighbors=3, weights="uniform").fit_transform(my_df[or_cols])
|
||||
, index = my_df['mutationinformation']
|
||||
, columns = or_cols )
|
||||
my_dfI.columns = ['or_rawI', 'logorI']
|
||||
|
|
@ -236,15 +388,15 @@ def setvars(gene,drug):
|
|||
#-------------------------------------------
|
||||
# OR df Merge: with original based on index
|
||||
#-------------------------------------------
|
||||
my_df['index_bm'] = my_df.index
|
||||
#my_df['index_bm'] = my_df.index
|
||||
mydf_imputed = pd.merge(my_df
|
||||
, my_dfI
|
||||
, on = 'mutationinformation')
|
||||
mydf_imputed = mydf_imputed.set_index(['index_bm'])
|
||||
#mydf_imputed = mydf_imputed.set_index(['index_bm'])
|
||||
|
||||
my_df['log10_or_mychisq'].isna().sum()
|
||||
mydf_imputed['log10_or_mychisq'].isna().sum()
|
||||
mydf_imputed['logorI'].isna().sum()
|
||||
mydf_imputed['logorI'].isna().sum() # should be 0
|
||||
|
||||
len(my_df.columns)
|
||||
len(mydf_imputed.columns)
|
||||
|
|
@ -254,6 +406,17 @@ def setvars(gene,drug):
|
|||
#-----------------------------------------
|
||||
my_df = mydf_imputed.copy()
|
||||
|
||||
if my_df['logorI'].isna().sum() == 0:
|
||||
print('\nPASS: OR values imputed, data ready for ML')
|
||||
else:
|
||||
sys.exit('\nFAIL: something went wrong, Data not ready for ML. Please check upstream!')
|
||||
|
||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
#---------------------------------------
|
||||
# TODO: try other imputation like MICE
|
||||
#---------------------------------------
|
||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
#%%########################################################################
|
||||
#==========================
|
||||
# Data for ML
|
||||
|
|
@ -347,11 +510,17 @@ def setvars(gene,drug):
|
|||
|
||||
X_genomicFN = X_genomic_mafor + X_genomic_linegae
|
||||
|
||||
X_aaindexFN = list(aa_df_cols)
|
||||
|
||||
print('\nTotal no. of features for aaindex:', len(X_aaindexFN))
|
||||
|
||||
#%% Construct numerical and categorical column names
|
||||
# numerical feature names
|
||||
# numerical_FN = common_cols_stabiltyN + foldX_cols + X_strFN + X_evolFN + X_genomicFN
|
||||
|
||||
numerical_FN = X_ssFN + X_evolFN + X_genomicFN
|
||||
#numerical_FN = X_ssFN + X_evolFN + X_genomicFN
|
||||
numerical_FN = X_ssFN + X_evolFN + X_genomicFN + X_aaindexFN
|
||||
|
||||
|
||||
#categorical feature names
|
||||
categorical_FN = ['ss_class'
|
||||
|
|
@ -365,9 +534,17 @@ def setvars(gene,drug):
|
|||
, 'electrostatics_change'
|
||||
, 'polarity_change'
|
||||
, 'water_change'
|
||||
, 'drtype_mode_labels' # beware then you can use it to predict
|
||||
#, 'active_aa_pos' # TODO?
|
||||
#, 'drtype_mode_labels' # beware then you can't use it to predict [USED it for uq_v1]
|
||||
, 'active_site'
|
||||
#, 'gene_name' # will be required for the combined stuff
|
||||
]
|
||||
#----------------------------------------------
|
||||
# count numerical and categorical features
|
||||
#----------------------------------------------
|
||||
|
||||
print('\nNo. of numerical features:', len(numerical_FN)
|
||||
, '\nNo. of categorical features:', len(categorical_FN))
|
||||
|
||||
###########################################################################
|
||||
#=======================
|
||||
# Masking columns:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue