adde script to run ml baseline models orig version with feature groups
This commit is contained in:
parent
137f19a285
commit
fe0986aa28
2 changed files with 562 additions and 4 deletions
|
|
@ -419,7 +419,7 @@ def setvars(gene,drug):
|
|||
#---------------------------------------
|
||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
#%% Data for ML ###############################################################
|
||||
#%% Data for ML
|
||||
#==========================
|
||||
# Data for ML
|
||||
#==========================
|
||||
|
|
@ -428,7 +428,7 @@ def setvars(gene,drug):
|
|||
# Build column names to mask for affinity chanhes
|
||||
if gene.lower() in geneL_basic:
|
||||
#X_stabilityN = common_cols_stabiltyN
|
||||
gene_affinity_colnames = []# not needed as its a common one
|
||||
gene_affinity_colnames = []# not needed as its the common ones
|
||||
cols_to_mask = ['ligand_affinity_change']
|
||||
|
||||
if gene.lower() in geneL_ppi2:
|
||||
|
|
@ -487,7 +487,6 @@ def setvars(gene,drug):
|
|||
, 'ddg_foldx'
|
||||
, 'deepddg'
|
||||
, 'ddg_dynamut2'
|
||||
, 'mmcsm_lig'
|
||||
, 'contacts']
|
||||
#--------
|
||||
# FoldX
|
||||
|
|
@ -506,7 +505,8 @@ def setvars(gene,drug):
|
|||
# FG3: Affinity features
|
||||
#===================
|
||||
common_affinity_Fnum = ['ligand_distance'
|
||||
, 'ligand_affinity_change']
|
||||
, 'ligand_affinity_change'
|
||||
, 'mmcsm_lig']
|
||||
|
||||
# if gene.lower() in geneL_basic:
|
||||
# X_affinityFN = common_affinity_Fnum
|
||||
|
|
|
|||
558
scripts/ml/run_fg.py
Executable file
558
scripts/ml/run_fg.py
Executable file
|
|
@ -0,0 +1,558 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Sat May 28 05:25:30 2022
|
||||
|
||||
@author: tanu
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
|
||||
###############################################################################
|
||||
# gene = 'pncA'
|
||||
# drug = 'pyrazinamide'
|
||||
#total_mtblineage_uc = 8
|
||||
|
||||
#%% command line args: case sensitive
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
arg_parser.add_argument('-d', '--drug', help = 'drug name', default = '')
|
||||
arg_parser.add_argument('-g', '--gene', help = 'gene name', default = '')
|
||||
args = arg_parser.parse_args()
|
||||
|
||||
drug = args.drug
|
||||
gene = args.gene
|
||||
###############################################################################
|
||||
homedir = os.path.expanduser("~")
|
||||
os.chdir( homedir + '/git/LSHTM_analysis/scripts/ml/')
|
||||
|
||||
#==================
|
||||
# Import data
|
||||
#==================
|
||||
from ml_data_dissected import *
|
||||
setvars(gene,drug)
|
||||
from ml_data_dissected import *
|
||||
|
||||
# from YC run_all_ML: run locally
|
||||
#from UQ_yc_RunAllClfs import run_all_ML
|
||||
|
||||
#====================
|
||||
# Import ML function
|
||||
#====================
|
||||
# TT run all ML clfs: baseline model
|
||||
from MultModelsCl import MultModelsCl
|
||||
|
||||
############################################################################
|
||||
print('\n#####################################################################\n'
|
||||
, '\nRunning ML analysis: feature groups '
|
||||
, '\nGene name:', gene
|
||||
, '\nDrug name:', drug)
|
||||
|
||||
###############################################################################
|
||||
#==================
|
||||
# Specify outdir
|
||||
#==================
|
||||
outdir_ml = outdir + 'ml/uq_v1/fgs/'
|
||||
print('\nOutput directory:', outdir_ml)
|
||||
outFile = outdir_ml + gene.lower() + '_baseline_FG.csv'
|
||||
|
||||
#==================
|
||||
# other vars
|
||||
#==================
|
||||
tts_split_name = 'original'
|
||||
resampling = 'none'
|
||||
|
||||
###############################################################################
|
||||
score_type_ordermapD = { 'mcc' : 1
|
||||
, 'fscore' : 2
|
||||
, 'jcc' : 3
|
||||
, 'precision' : 4
|
||||
, 'recall' : 5
|
||||
, 'accuracy' : 6
|
||||
, 'roc_auc' : 7
|
||||
, 'TN' : 8
|
||||
, 'FP' : 9
|
||||
, 'FN' : 10
|
||||
, 'TP' : 11
|
||||
, 'trainingY_neg': 12
|
||||
, 'trainingY_pos': 13
|
||||
, 'blindY_neg' : 14
|
||||
, 'blindY_pos' : 15
|
||||
, 'fit_time' : 16
|
||||
, 'score_time' : 17
|
||||
}
|
||||
#%%###########################################################################
|
||||
print('\n================================================================\n')
|
||||
|
||||
#all_featuresN = X_evolFN + X_structural_FN + X_genomicFN
|
||||
# X_structural_FN = X_stability_FN + X_affinityFN + X_resprop_FN
|
||||
# X_resprop_FN = X_aaindex_Fnum + X_str_Fnum + X_aap_Fcat
|
||||
|
||||
print('\n================================================================'
|
||||
|
||||
, '\nTotal Evolutionary features (n):' , len(X_evolFN)
|
||||
, '\n--------------Evol. feature colnames:', X_evolFN
|
||||
|
||||
, '\n================================================================'
|
||||
|
||||
, '\n\nTotal structural features (n):', len(X_structural_FN)
|
||||
|
||||
, '\n--------Stability ncols:' , len(X_stability_FN)
|
||||
, '\n--------------Common stability colnames:' , X_common_stability_Fnum
|
||||
, '\n--------------Foldx colnames:' , X_foldX_Fnum
|
||||
|
||||
, '\n--------Affinity ncols:' , len(X_affinityFN)
|
||||
, '\n--------------Common affinity colnames:' , common_affinity_Fnum
|
||||
, '\n--------------Gene specific affinity colnames:', gene_affinity_colnames
|
||||
|
||||
, '\n--------Residue prop ncols:' , len(X_resprop_FN)
|
||||
, '\n--------------Residue Prop cols:' , X_str_Fnum
|
||||
, '\n--------------AA change Prop cols:' , X_aap_Fcat
|
||||
, '\n--------------AA index cols:' , X_aaindex_Fnum
|
||||
|
||||
, '\n================================================================'
|
||||
|
||||
, '\n\nTotal Genomic features (n):' , len(X_genomicFN)
|
||||
, '\n--------MAF+OR cols:' , len(X_gn_mafor_Fnum)
|
||||
, '\n--------------MAF+OR colnames:' , X_gn_mafor_Fnum
|
||||
|
||||
, '\n--------Lineage cols:' , len(X_gn_linegae_Fnum)
|
||||
, '\n--------------Lineage cols:' , X_gn_linegae_Fnum
|
||||
|
||||
, '\n--------Other cols:' , len(X_gn_Fcat)
|
||||
, '\n--------------Other cols:' , X_gn_Fcat
|
||||
|
||||
, '\n================================================================')
|
||||
|
||||
# Sanity check
|
||||
if ( len(X.columns) == len(X_evolFN) + len(X_structural_FN) + len(X_genomicFN)):
|
||||
print('\nPass: No. of features match')
|
||||
else:
|
||||
print('\nFail: Count of feature mismatch'
|
||||
, '\nExpected:', len(X_evolFN) + len(X_structural_FN) + len(X_genomicFN)
|
||||
, '\nGot:', len(X.columns))
|
||||
sys.exit()
|
||||
|
||||
print('\n#####################################################################\n')
|
||||
###############################################################################
|
||||
#================
|
||||
# Evolutionary
|
||||
# X_evolFN
|
||||
#================
|
||||
feature_gp_nameEV = 'evolutionary'
|
||||
n_featuresEV = len(X_evolFN)
|
||||
|
||||
scores_mmEV = MultModelsCl(input_df = X[X_evolFN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_evolFN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allEV = pd.DataFrame(scores_mmEV)
|
||||
|
||||
baseline_EV = baseline_allEV.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_EV = baseline_EV.reset_index()
|
||||
baseline_EV.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_EV['data_source'] = baseline_EV.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_EV['score_type'] = baseline_EV['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_EV['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_EV['score_order'] = baseline_EV['score_type'].map(score_type_ordermapD)
|
||||
baseline_EV.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_EV['feature_group'] = feature_gp_nameEV
|
||||
baseline_EV['resampling'] = resampling
|
||||
baseline_EV['tts_split'] = tts_split_name
|
||||
baseline_EV['n_features'] = n_featuresEV
|
||||
###############################################################################
|
||||
#================
|
||||
# Genomics
|
||||
# X_genomicFN
|
||||
#================
|
||||
feature_gp_nameGN = 'genomics'
|
||||
n_featuresGN = len(X_genomicFN)
|
||||
|
||||
scores_mmGN = MultModelsCl(input_df = X[X_genomicFN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_genomicFN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allGN = pd.DataFrame(scores_mmGN)
|
||||
|
||||
baseline_GN = baseline_allGN.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_GN = baseline_GN.reset_index()
|
||||
baseline_GN.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_GN['data_source'] = baseline_GN.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_GN['score_type'] = baseline_GN['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_GN['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_GN['score_order'] = baseline_GN['score_type'].map(score_type_ordermapD)
|
||||
baseline_GN.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_GN['feature_group'] = feature_gp_nameGN
|
||||
baseline_GN['resampling'] = resampling
|
||||
baseline_GN['tts_split'] = tts_split_name
|
||||
baseline_GN['n_features'] = n_featuresGN
|
||||
###############################################################################
|
||||
#all_featuresN = X_evolFN + X_structural_FN + X_genomicFN
|
||||
# X_structural_FN = X_stability_FN + X_affinityFN + X_resprop_FN
|
||||
# X_resprop_FN = X_aaindex_Fnum + X_str_Fnum + X_aap_Fcat
|
||||
#================
|
||||
# Structural cols
|
||||
# X_structural_FN
|
||||
#================
|
||||
feature_gp_nameSTR = 'structural'
|
||||
n_featuresSTR = len(X_structural_FN)
|
||||
|
||||
scores_mmSTR = MultModelsCl(input_df = X[X_structural_FN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_structural_FN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allSTR = pd.DataFrame(scores_mmSTR)
|
||||
|
||||
baseline_STR = baseline_allSTR.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_STR = baseline_STR.reset_index()
|
||||
baseline_STR.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_STR['data_source'] = baseline_STR.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_STR['score_type'] = baseline_STR['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_STR['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_STR['score_order'] = baseline_STR['score_type'].map(score_type_ordermapD)
|
||||
baseline_STR.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_STR['feature_group'] = feature_gp_nameSTR
|
||||
baseline_STR['resampling'] = resampling
|
||||
baseline_STR['tts_split'] = tts_split_name
|
||||
baseline_STR['n_features'] = n_featuresSTR
|
||||
##############################################################################
|
||||
#================
|
||||
# Stability cols
|
||||
# X_stability_FN
|
||||
#================
|
||||
feature_gp_nameSTB = 'stability'
|
||||
n_featuresSTB = len(X_stability_FN)
|
||||
|
||||
scores_mmSTB = MultModelsCl(input_df = X[X_stability_FN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_stability_FN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allSTB = pd.DataFrame(scores_mmSTB)
|
||||
|
||||
baseline_STB = baseline_allSTB.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_STB = baseline_STB.reset_index()
|
||||
baseline_STB.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_STB['data_source'] = baseline_STB.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_STB['score_type'] = baseline_STB['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_STB['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_STB['score_order'] = baseline_STB['score_type'].map(score_type_ordermapD)
|
||||
baseline_STB.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_STB['feature_group'] = feature_gp_nameSTB
|
||||
baseline_STB['resampling'] = resampling
|
||||
baseline_STB['tts_split'] = tts_split_name
|
||||
baseline_STB['n_features'] = n_featuresSTB
|
||||
###############################################################################
|
||||
#================
|
||||
# Affinity cols
|
||||
# X_affinityFN
|
||||
#================
|
||||
feature_gp_nameAFF = 'affinity'
|
||||
n_featuresAFF = len(X_affinityFN)
|
||||
|
||||
scores_mmAFF = MultModelsCl(input_df = X[X_affinityFN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_affinityFN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allAFF = pd.DataFrame(scores_mmAFF)
|
||||
|
||||
baseline_AFF = baseline_allAFF.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_AFF = baseline_AFF.reset_index()
|
||||
baseline_AFF.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_AFF['data_source'] = baseline_AFF.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_AFF['score_type'] = baseline_AFF['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_AFF['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_AFF['score_order'] = baseline_AFF['score_type'].map(score_type_ordermapD)
|
||||
baseline_AFF.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_AFF['feature_group'] = feature_gp_nameAFF
|
||||
baseline_AFF['resampling'] = resampling
|
||||
baseline_AFF['tts_split'] = tts_split_name
|
||||
baseline_AFF['n_features'] = n_featuresAFF
|
||||
###############################################################################
|
||||
#================
|
||||
# Residue level
|
||||
# X_resprop_FN
|
||||
#================
|
||||
feature_gp_nameRES = 'residue_prop'
|
||||
n_featuresRES = len(X_resprop_FN)
|
||||
|
||||
scores_mmRES = MultModelsCl(input_df = X[X_resprop_FN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_resprop_FN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allRES = pd.DataFrame(scores_mmRES)
|
||||
|
||||
baseline_RES = baseline_allRES.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_RES = baseline_RES.reset_index()
|
||||
baseline_RES.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_RES['data_source'] = baseline_RES.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_RES['score_type'] = baseline_RES['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_RES['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_RES['score_order'] = baseline_RES['score_type'].map(score_type_ordermapD)
|
||||
baseline_RES.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_RES['feature_group'] = feature_gp_nameRES
|
||||
baseline_RES['resampling'] = resampling
|
||||
baseline_RES['tts_split'] = tts_split_name
|
||||
baseline_RES['n_features'] = n_featuresRES
|
||||
###############################################################################
|
||||
#================
|
||||
# Residue level-AAindex
|
||||
#X_resprop_FN - X_aaindex_Fnum
|
||||
#================
|
||||
X_respropNOaaFN = list(set(X_resprop_FN) - set(X_aaindex_Fnum))
|
||||
|
||||
feature_gp_nameRNAA = 'ResPropNoAA'
|
||||
n_featuresRNAA = len(X_respropNOaaFN)
|
||||
|
||||
scores_mmRNAA = MultModelsCl(input_df = X[X_respropNOaaFN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_respropNOaaFN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allRNAA = pd.DataFrame(scores_mmRNAA)
|
||||
|
||||
baseline_RNAA = baseline_allRNAA.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_RNAA = baseline_RNAA.reset_index()
|
||||
baseline_RNAA.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_RNAA['data_source'] = baseline_RNAA.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_RNAA['score_type'] = baseline_RNAA['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_RNAA['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_RNAA['score_order'] = baseline_RNAA['score_type'].map(score_type_ordermapD)
|
||||
baseline_RNAA.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_RNAA['feature_group'] = feature_gp_nameRNAA
|
||||
baseline_RNAA['resampling'] = resampling
|
||||
baseline_RNAA['tts_split'] = tts_split_name
|
||||
baseline_RNAA['n_features'] = n_featuresRNAA
|
||||
###############################################################################
|
||||
#================
|
||||
# Structural cols-AAindex
|
||||
#X_structural_FN - X_aaindex_Fnum
|
||||
#================
|
||||
X_strNOaaFN = list(set(X_structural_FN) - set(X_aaindex_Fnum))
|
||||
|
||||
feature_gp_nameSNAA = 'StrNoAA'
|
||||
n_featuresSNAA = len(X_strNOaaFN)
|
||||
|
||||
scores_mmSNAA = MultModelsCl(input_df = X[X_strNOaaFN]
|
||||
, target = y
|
||||
, var_type = 'mixed'
|
||||
, skf_cv = skf_cv
|
||||
, blind_test_input_df = X_bts[X_strNOaaFN]
|
||||
, blind_test_target = y_bts
|
||||
, add_cm = True
|
||||
, add_yn = True)
|
||||
|
||||
baseline_allSNAA = pd.DataFrame(scores_mmSNAA)
|
||||
|
||||
baseline_SNAA = baseline_allSNAA.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
|
||||
baseline_SNAA = baseline_SNAA.reset_index()
|
||||
baseline_SNAA.rename(columns = {'index': 'original_names'}, inplace = True)
|
||||
|
||||
# Indicate whether BT or CT
|
||||
bt_pattern = re.compile(r'bts_.*')
|
||||
baseline_SNAA['data_source'] = baseline_SNAA.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
|
||||
|
||||
baseline_SNAA['score_type'] = baseline_SNAA['original_names'].str.replace('bts_|test_', '', regex = True)
|
||||
|
||||
score_type_uniqueN = set(baseline_SNAA['score_type'])
|
||||
cL1 = list(score_type_ordermapD.keys())
|
||||
cL2 = list(score_type_uniqueN)
|
||||
|
||||
if set(cL1).issubset(cL2):
|
||||
print('\nPASS: sorting df by score that is mapped onto the order I want')
|
||||
baseline_SNAA['score_order'] = baseline_SNAA['score_type'].map(score_type_ordermapD)
|
||||
baseline_SNAA.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
|
||||
else:
|
||||
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
|
||||
|
||||
baseline_SNAA['feature_group'] = feature_gp_nameSNAA
|
||||
baseline_SNAA['resampling'] = resampling
|
||||
baseline_SNAA['tts_split'] = tts_split_name
|
||||
baseline_SNAA['n_features'] = n_featuresSNAA
|
||||
###############################################################################
|
||||
#%% COMBINING all FG dfs
|
||||
#================
|
||||
# Combine all
|
||||
# https://stackoverflow.com/questions/39862654/pandas-concat-of-multiple-data-frames-using-only-common-columns
|
||||
#================
|
||||
dfs_combine = [baseline_EV, baseline_GN, baseline_STR, baseline_STB, baseline_AFF, baseline_RES , baseline_RNAA , baseline_SNAA]
|
||||
|
||||
dfs_nrows = []
|
||||
for df in dfs_combine:
|
||||
dfs_nrows = dfs_nrows + [len(df)]
|
||||
dfs_nrows = max(dfs_nrows)
|
||||
|
||||
dfs_ncols = []
|
||||
for df in dfs_combine:
|
||||
dfs_ncols = dfs_ncols + [len(df.columns)]
|
||||
dfs_ncols = max(dfs_ncols)
|
||||
|
||||
# dfs_ncols = []
|
||||
# dfs_ncols2 = mode(dfs_ncols.append(len(df.columns) for df in dfs_combine)
|
||||
# dfs_ncols2
|
||||
|
||||
expected_nrows = len(dfs_combine) * dfs_nrows
|
||||
expected_ncols = dfs_ncols
|
||||
|
||||
common_cols = list(set.intersection(*(set(df.columns) for df in dfs_combine)))
|
||||
|
||||
if len(common_cols) == dfs_ncols :
|
||||
combined_FG_baseline = pd.concat([df[common_cols] for df in dfs_combine], ignore_index=True)
|
||||
fgs = combined_FG_baseline[['feature_group', 'n_features']]
|
||||
fgs = fgs.drop_duplicates()
|
||||
print('\nConcatenating dfs with feature groups after ML analysis (sampling type):'
|
||||
, '\nNo. of dfs combining:', len(dfs_combine)
|
||||
, '\nSampling type:', resampling
|
||||
, '\nThe feature groups are:'
|
||||
, '\n', fgs)
|
||||
if len(combined_FG_baseline) == expected_nrows and len(combined_FG_baseline.columns) == expected_ncols:
|
||||
print('\nPASS:', len(dfs_combine), 'dfs successfully combined'
|
||||
, '\nnrows in combined_df:', len(combined_FG_baseline)
|
||||
, '\nncols in combined_df:', len(combined_FG_baseline.columns))
|
||||
else:
|
||||
print('\nFAIL: concatenating failed'
|
||||
, '\nExpected nrows:', expected_nrows
|
||||
, '\nGot:', len(combined_FG_baseline)
|
||||
, '\nExpected ncols:', expected_ncols
|
||||
, '\nGot:', len(combined_FG_baseline.columns))
|
||||
sys.exit()
|
||||
else:
|
||||
sys.exit('\nConcatenting dfs not possible,check numbers ')
|
||||
|
||||
# # rpow bind
|
||||
# if all(ll((baseline_EV.columns == baseline_GN.columns == baseline_STR.columns)):
|
||||
# print('\nPASS:colnames match, proceeding to rowbind')
|
||||
# comb_df = pd.concat()], axis = 0, ignore_index = True )
|
||||
###############################################################################
|
||||
#====================
|
||||
# Write output file
|
||||
#====================
|
||||
|
||||
combined_FG_baseline.to_csv(outFile, index = False)
|
||||
print('\nFile successfully written:', outFile)
|
||||
###############################################################################
|
||||
Loading…
Add table
Add a link
Reference in a new issue