tanu/LSHTM_analysis
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

changed ml output dirs and ready to run fs

Browse source
This commit is contained in:
Tanushree Tunstall 2022-07-01 21:40:14 +01:00
parent 57348f1874
commit 11af00f1db
5 changed files with 67 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

8
scripts/ml/combined_model/cm_logo_skf.py
View file

@ -80,6 +80,8 @@ homedir = os.path.expanduser("~")
sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions')
sys.path sys.path
############################################################################### ###############################################################################
outdir = homedir + '/git/LSHTM_ML/output/combined/
#==================== #====================
# Import ML functions # Import ML functions
#==================== #====================
@ -92,6 +94,9 @@ skf_cv = StratifiedKFold(n_splits = 10 , shuffle = True, random_state = 42)
#logo = LeaveOneGroupOut() #logo = LeaveOneGroupOut()
########################################################################
# COMPLETE data: No tts_split
########################################################################
#%% #%%
def CMLogoSkf(combined_df def CMLogoSkf(combined_df
, all_genes = ["embb", "katg", "rpob", "pnca", "gid", "alr"] , all_genes = ["embb", "katg", "rpob", "pnca", "gid", "alr"]
@ -125,7 +130,8 @@ def CMLogoSkf(combined_df
tts_split_type = "logo_skf_BT_" + bts_gene tts_split_type = "logo_skf_BT_" + bts_gene
outFile = "/home/tanu/git/Data/ml_combined/" + str(n_tr_genes+1) + "genes_" + tts_split_type + ".csv" outFile = outdir + str(n_tr_genes+1) + "genes_" + tts_split_type + ".csv"
print(outFile) print(outFile)
#------- #-------

26
scripts/ml/combined_model/cm_ml_iterator.py
View file

@ -15,19 +15,19 @@ homedir = os.path.expanduser("~")
sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions')
sys.path sys.path
############################################################################### ###############################################################################
outdir = homedir + '/git/LSHTM_ML/output/combined/
#==================== #====================
# Import ML functions # Import ML functions
#==================== #====================
from MultClfs import * #from MultClfs import *
from MultClfs_logo_skf import *
from GetMLData import * from GetMLData import *
from SplitTTS import * from SplitTTS import *
# param dict for getmldata() # Input data
combined_model_paramD = {'data_combined_model' : False from ml_data_combined import *
, 'use_or' : False
, 'omit_all_genomic_features': False
, 'write_maskfile' : False
, 'write_outfile' : False }
############################################################################### ###############################################################################
#ml_genes = ["pncA", "embB", "katG", "rpoB", "gid"] #ml_genes = ["pncA", "embB", "katG", "rpoB", "gid"]
@ -54,7 +54,7 @@ for gene, drug in ml_gene_drugD.items():
for split_type in split_types: for split_type in split_types:
for data_type in split_data_types: for data_type in split_data_types:
out_filename = (gene.lower()+'_'+split_type+'_'+data_type+'.csv') out_filename = outdir + gene.lower()+ '_' + split_type + '_' + data_type + '.csv'
tempD=split_tts(gene_dataD[gene_low] tempD=split_tts(gene_dataD[gene_low]
, data_type = data_type , data_type = data_type
, split_type = split_type , split_type = split_type
@ -88,14 +88,8 @@ for gene, drug in ml_gene_drugD.items():
mmDD = {} mmDD = {}
for k, v in paramD.items(): for k, v in paramD.items():
scoresD = MultModelsCl(**paramD[k] scoresD = MultModelsCl_logo_skf(**paramD[k]
, tts_split_type = split_type XXXXXXXXXXXXXXXXXXXXXXX
, skf_cv = skf_cv
, blind_test_df = tempD['X_bts']
, blind_test_target = tempD['y_bts']
, add_cm = True
, add_yn = True
, return_formatted_output = True)
mmDD[k] = scoresD mmDD[k] = scoresD
# Extracting the dfs from within the dict and concatenating to output as one df # Extracting the dfs from within the dict and concatenating to output as one df

8
scripts/ml/ml_iterator.py
View file

@ -15,6 +15,8 @@ homedir = os.path.expanduser("~")
sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions')
sys.path sys.path
############################################################################### ###############################################################################
outdir = homedir + '/git/LSHTM_ML/output/genes/'
#==================== #====================
# Import ML functions # Import ML functions
#==================== #====================
@ -54,7 +56,9 @@ for gene, drug in ml_gene_drugD.items():
for split_type in split_types: for split_type in split_types:
for data_type in split_data_types: for data_type in split_data_types:
out_filename = (gene.lower()+'_'+split_type+'_'+data_type+'.csv')
out_filename = outdir + gene.lower() + '_' + split_type + '_' + data_type + '.csv'
tempD=split_tts(gene_dataD[gene_low] tempD=split_tts(gene_dataD[gene_low]
, data_type = data_type , data_type = data_type
, split_type = split_type , split_type = split_type
@ -103,5 +107,5 @@ for gene, drug in ml_gene_drugD.items():
out_wf= pd.concat(mmDD, ignore_index = True) out_wf= pd.concat(mmDD, ignore_index = True)
out_wf_f = out_wf.sort_values(by = ['resampling', 'source_data', 'MCC'], ascending = [True, True, False], inplace = False) out_wf_f = out_wf.sort_values(by = ['resampling', 'source_data', 'MCC'], ascending = [True, True, False], inplace = False)
out_wf_f.to_csv(('/home/tanu/git/Data/ml_combined/genes/'+out_filename), index = False) out_wf_f.to_csv(('/home/tanu/git/Data/ml_combined/genes/'+ out_filename), index = False)

74
scripts/ml/ml_iterator_fs.py Normal file → Executable file
View file

@ -15,6 +15,8 @@ homedir = os.path.expanduser("~")
sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions') sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions')
sys.path sys.path
############################################################################### ###############################################################################
outdir = homedir + '/git/LSHTM_ML/output/fs/'
#==================== #====================
# Import ML functions # Import ML functions
#==================== #====================
@ -31,7 +33,8 @@ combined_model_paramD = {'data_combined_model' : False
, 'write_outfile' : False } , 'write_outfile' : False }
############################################################################### ###############################################################################
#ml_genes = ["pncA", "embB", "katG", "rpoB", "gid"] #ml_genes = ["pncA", "embB", "katG", "rpoB", "gid"]
outdir = homedir + '/git/Data/ml_combined/fs/' # outdir = homedir + '/git/Data/ml_combined/fs/'
ml_gene_drugD = {'pncA' : 'pyrazinamide' ml_gene_drugD = {'pncA' : 'pyrazinamide'
# , 'embB' : 'ethambutol' # , 'embB' : 'ethambutol'
# , 'katG' : 'isoniazid' # , 'katG' : 'isoniazid'
@ -39,26 +42,27 @@ ml_gene_drugD = {'pncA' : 'pyrazinamide'
# , 'gid' : 'streptomycin' # , 'gid' : 'streptomycin'
} }
gene_dataD={} gene_dataD={}
#split_types = ['70_30', '80_20', 'sl'] split_types = ['70_30', '80_20', 'sl']
#split_data_types = ['actual', 'complete']
split_types = ['70_30']
split_data_types = ['actual', 'complete'] split_data_types = ['actual', 'complete']
#split_types = ['70_30']
#split_data_types = ['actual', 'complete']
fs_models = [('Logistic Regression' , LogisticRegression(**rs) )] #fs_models = [('Logistic Regression' , LogisticRegression(**rs) )]
# fs_models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) )
# , ('Decision Tree' , DecisionTreeClassifier(**rs) ) fs_models = [('AdaBoost Classifier' , AdaBoostClassifier(**rs) )
# , ('Extra Tree' , ExtraTreeClassifier(**rs) ) , ('Decision Tree' , DecisionTreeClassifier(**rs) )
# , ('Extra Trees' , ExtraTreesClassifier(**rs) ) , ('Extra Tree' , ExtraTreeClassifier(**rs) )
# , ('Gradient Boosting' , GradientBoostingClassifier(**rs) ) , ('Extra Trees' , ExtraTreesClassifier(**rs) )
# , ('LDA' , LinearDiscriminantAnalysis() ) , ('Gradient Boosting' , GradientBoostingClassifier(**rs) )
# , ('Logistic Regression' , LogisticRegression(**rs) ) , ('LDA' , LinearDiscriminantAnalysis() )
# , ('Logistic RegressionCV' , LogisticRegressionCV(cv = 3, **rs)) , ('Logistic Regression' , LogisticRegression(**rs) )
# , ('Passive Aggresive' , PassiveAggressiveClassifier(**rs, **njobs) ) , ('Logistic RegressionCV' , LogisticRegressionCV(cv = 3, **rs))
# , ('Random Forest' , RandomForestClassifier(**rs, n_estimators = 1000 ) ) , ('Passive Aggresive' , PassiveAggressiveClassifier(**rs, **njobs) )
# , ('Ridge Classifier' , RidgeClassifier(**rs) ) , ('Random Forest' , RandomForestClassifier(**rs, n_estimators = 1000 ) )
# , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) ) , ('Ridge Classifier' , RidgeClassifier(**rs) )
# , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) ) , ('Ridge ClassifierCV' , RidgeClassifierCV(cv = 3) )
# ] , ('Stochastic GDescent' , SGDClassifier(**rs, **njobs) )
]
for gene, drug in ml_gene_drugD.items(): for gene, drug in ml_gene_drugD.items():
print ('\nGene:', gene print ('\nGene:', gene
@ -88,25 +92,27 @@ for gene, drug in ml_gene_drugD.items():
, 'target' : tempD['y'] , 'target' : tempD['y']
, 'var_type' : 'mixed' , 'var_type' : 'mixed'
, 'resampling_type': 'none'} , 'resampling_type': 'none'}
,'smnc_paramD': { 'input_df' : tempD['X_smnc']
, 'smnc_paramD' : { 'input_df' : tempD['X_smnc']
, 'target' : tempD['y_smnc'] , 'target' : tempD['y_smnc']
, 'var_type' : 'mixed' , 'var_type' : 'mixed'
, 'resampling_type' : 'smnc'} , 'resampling_type' : 'smnc'}
# , 'ros_paramD': { 'input_df' : tempD['X_ros']
# , 'target' : tempD['y_ros'] , 'ros_paramD' : { 'input_df' : tempD['X_ros']
# , 'var_type' : 'mixed' , 'target' : tempD['y_ros']
# , 'resampling_type' : 'ros'} , 'var_type' : 'mixed'
# , 'rus_paramD' : { 'input_df' : tempD['X_rus'] , 'resampling_type' : 'ros'}
# , 'target' : tempD['y_rus']
# , 'var_type' : 'mixed' , 'rus_paramD' : { 'input_df' : tempD['X_rus']
# , 'resampling_type' : 'rus'} , 'target' : tempD['y_rus']
# , 'rouC_paramD' : { 'input_df' : tempD['X_rouC'] , 'var_type' : 'mixed'
# , 'target' : tempD['y_rouC'] , 'resampling_type' : 'rus'}
# , 'var_type' : 'mixed'
# , 'resampling_type': 'rouC'} , 'rouC_paramD' : { 'input_df' : tempD['X_rouC']
, 'target' : tempD['y_rouC']
, 'var_type' : 'mixed'
, 'resampling_type': 'rouC'}
} }
#for m in fs_models:
# print(m)
out_fsD = {} out_fsD = {}
index = 1 index = 1

95
scripts/ml/running_ml_scripts.txt
View file

@ -1,95 +0,0 @@
########################################################################
# 70/30 [WITHOUT OR]
########################################################################
=-----------------------------------=
# actual data
#------------------------------------=
time ./run_7030.py -g pncA -d pyrazinamide 2>&1 | tee log_pnca_7030_.txt
time ./run_7030.py -g embB -d ethambutol 2>&1 | tee log_embb_7030_.txt
time ./run_7030.py -g katG -d isoniazid 2>&1 | tee log_katg_7030_.txt
time ./run_7030.py -g rpoB -d rifampicin 2>&1 | tee log_rpob_7030_.txt
time ./run_7030.py -g gid -d streptomycin 2>&1 | tee log_gid_7030_.txt
time ./run_7030.py -g alr -d cycloserine 2>&1 | tee log_alr_7030_.txt
=-----------------------------------=
# COMPLETE data
#------------------------------------=
time ./run_cd_7030.py -g pncA -d pyrazinamide 2>&1 | tee log_pnca_cd_7030_.txt
time ./run_cd_7030.py -g embB -d ethambutol 2>&1 | tee log_embb_cd_7030_.txt
time ./run_cd_7030.py -g katG -d isoniazid 2>&1 | tee log_katg_cd_7030_.txt
time ./run_cd_7030.py -g rpoB -d rifampicin 2>&1 | tee log_rpob_cd_7030_.txt
time ./run_cd_7030.py -g gid -d streptomycin 2>&1 | tee log_gid_cd_7030_.txt
time ./run_cd_7030.py -g alr -d cycloserine 2>&1 | tee log_alr_cd_7030_.txt
########################################################################
# 80/20 [WITHOUT OR]
########################################################################
=-----------------------------------=
# actual data
#------------------------------------=
time ./run_8020.py -g pncA -d pyrazinamide 2>&1 | tee log_pnca_8020_.txt
time ./run_8020.py -g embB -d ethambutol 2>&1 | tee log_embb_8020_.txt
time ./run_8020.py -g katG -d isoniazid 2>&1 | tee log_katg_8020_.txt
time ./run_8020.py -g rpoB -d rifampicin 2>&1 | tee log_rpob_8020_.txt
time ./run_8020.py -g gid -d streptomycin 2>&1 | tee log_gid_8020_.txt
time ./run_8020.py -g alr -d cycloserine 2>&1 | tee log_alr_8020_.txt
=-----------------------------------=
# COMPLETE data
#------------------------------------=
time ./run_cd_8020.py -g pncA -d pyrazinamide 2>&1 | tee log_pnca_cd_8020_.txt
time ./run_cd_8020.py -g embB -d ethambutol 2>&1 | tee log_embb_cd_8020_.txt
time ./run_cd_8020.py -g katG -d isoniazid 2>&1 | tee log_katg_cd_8020_.txt
time ./run_cd_8020.py -g rpoB -d rifampicin 2>&1 | tee log_rpob_cd_8020_.txt
time ./run_cd_8020.py -g gid -d streptomycin 2>&1 | tee log_gid_cd_8020_.txt
time ./run_cd_8020.py -g alr -d cycloserine 2>&1 | tee log_alr_cd_8020_.txt
########################################################################
# SL [WITHOUT OR]
########################################################################
=-----------------------------------=
# actual data
#------------------------------------=
time ./run_sl.py -g pncA -d pyrazinamide 2>&1 | tee log_pnca_sl_.txt
time ./run_sl.py -g embB -d ethambutol 2>&1 | tee log_embb_sl_.txt
time ./run_sl.py -g katG -d isoniazid 2>&1 | tee log_katg_sl_.txt
time ./run_sl.py -g rpoB -d rifampicin 2>&1 | tee log_rpob_sl_.txt
time ./run_sl.py -g gid -d streptomycin 2>&1 | tee log_gid_sl_.txt
time ./run_sl.py -g alr -d cycloserine 2>&1 | tee log_alr_sl_.txt
=-----------------------------------=
# COMPLETE data
#------------------------------------=
time ./run_cd_sl.py -g pncA -d pyrazinamide 2>&1 | tee log_pnca_cd_sl_.txt
time ./run_cd_sl.py -g embB -d ethambutol 2>&1 | tee log_embb_cd_sl_.txt
time ./run_cd_sl.py -g katG -d isoniazid 2>&1 | tee log_katg_cd_sl_.txt
time ./run_cd_sl.py -g rpoB -d rifampicin 2>&1 | tee log_rpob_cd_sl_.txt
time ./run_cd_sl.py -g gid -d streptomycin 2>&1 | tee log_gid_cd_sl_.txt
time ./run_cd_sl.py -g alr -d cycloserine 2>&1 | tee log_alr_cd_sl_.txt
########################################################################
########################################################################
########################################################################
###################### Feature Selection ##########################
########################################################################
########################################################################
# 7030
time ./run_FS.py -g pncA -d pyrazinamide 2>&1 | tee log_FS_pnca_7030.txt
time ./run_FS_7030.py -g pncA -d pyrazinamide 2>&1 | tee log_FS_pnca_7030_.txt