tanu/LSHTM_analysis
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

saving work

Browse source
This commit is contained in:
Tanushree Tunstall 2022-06-24 13:21:21 +01:00
parent 3514e1b4ba
commit ad99efedd7
5 changed files with 46 additions and 507 deletions
Download patch file Download diff file Expand all files Collapse all files

20
scripts/ml/MultModelsCl.py
View file

@ -99,13 +99,11 @@ rskf_cv = RepeatedStratifiedKFold(n_splits = 10
mcc_score_fn = {'mcc': make_scorer(matthews_corrcoef)}
jacc_score_fn = {'jcc': make_scorer(jaccard_score)}
#FIXME
#====================
# Import ProcessFunc
#====================
from ProcessMultModelsCl import *
#from ProcessMultModelCl import *
#%%
# Multiple Classification - Model Pipeline
def MultModelsCl(input_df, target, skf_cv
@ -275,10 +273,10 @@ def MultModelsCl(input_df, target, skf_cv
btyn_pos = btyn[1]
# Build dict
tbtD = {'trainingY_neg' : tyn_neg
, 'trainingY_pos' : tyn_pos
, 'blindY_neg' : btyn_neg
, 'blindY_pos' : btyn_pos}
tbtD = {'n_trainingY_neg' : tyn_neg
, 'n_trainingY_pos' : tyn_pos
, 'n_blindY_neg' : btyn_neg
, 'n_blindY_pos' : btyn_pos}
#---------------------------------
# Update cv dict with cmD and tbtD
@ -339,11 +337,11 @@ def MultModelsCl(input_df, target, skf_cv
mm_skf_scoresD[model_name]['resampling'] = resampling_type
mm_skf_scoresD[model_name]['training_size'] = len(input_df)
mm_skf_scoresD[model_name]['trainingY_ratio'] = round(yc1_ratio, 2)
mm_skf_scoresD[model_name]['n_training_size'] = len(input_df)
mm_skf_scoresD[model_name]['n_trainingY_ratio'] = round(yc1_ratio, 2)
mm_skf_scoresD[model_name]['testSize'] = len(blind_test_df)
mm_skf_scoresD[model_name]['testY_ratio'] = round(yc2_ratio,2)
mm_skf_scoresD[model_name]['n_blind_test_size'] = len(blind_test_df)
mm_skf_scoresD[model_name]['n_testY_ratio'] = round(yc2_ratio,2)
mm_skf_scoresD[model_name]['n_features'] = len(input_df.columns)
mm_skf_scoresD[model_name]['tts_split'] = tts_split_type

467
scripts/ml/Mult_dissected_CALL.py
View file

@ -1,467 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 20 13:05:23 2022
@author: tanu
"""
import re
#all_featuresN = X_evolFN + X_structural_FN + X_genomicFN
# X_structural_FN = X_stability_FN + X_affinityFN + X_resprop_FN
# X_resprop_FN = X_aaindex_Fnum + X_str_Fnum + X_aap_Fcat
score_type_ordermapD = { 'mcc' : 1
, 'fscore' : 2
, 'jcc' : 3
, 'precision' : 4
, 'recall' : 5
, 'accuracy' : 6
, 'roc_auc' : 7
, 'TN' : 8
, 'FP' : 9
, 'FN' : 10
, 'TP' : 11
, 'trainingY_neg': 12
, 'trainingY_pos': 13
, 'blindY_neg' : 14
, 'blindY_pos' : 15
, 'fit_time' : 16
, 'score_time' : 17
}
###############################################################################
#==================
# Specify outdir
#==================
outdir_ml = outdir + 'ml/uq_v1/fgs/'
print('\nOutput directory:', outdir_ml)
outFile = outdir_ml + gene.lower() + '_baseline_FG.csv'
#==================
# other vars
#==================
tts_split_name = 'original'
sampling_type_name = 'none'
###############################################################################
#================
# Evolutionary
# X_evolFN
#================
feature_gp_nameEV = 'evolutionary'
n_featuresEV = len(X_evolFN)
scores_mmEV = MultModelsCl(input_df = X[X_evolFN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_evolFN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allEV = pd.DataFrame(scores_mmEV)
baseline_EV = baseline_allEV.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_EV = baseline_EV.reset_index()
baseline_EV.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_EV['data_source'] = baseline_EV.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_EV['score_type'] = baseline_EV['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_EV['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_EV['score_order'] = baseline_EV['score_type'].map(score_type_ordermapD)
baseline_EV.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_EV['feature_group'] = feature_gp_nameEV
baseline_EV['sampling_type'] = sampling_type_name
baseline_EV['tts_split'] = tts_split_name
baseline_EV['n_features'] = n_featuresEV
###############################################################################
#================
# Genomics
# X_genomicFN
#================
feature_gp_nameGN = 'genomics'
n_featuresGN = len(X_genomicFN)
scores_mmGN = MultModelsCl(input_df = X[X_genomicFN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_genomicFN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allGN = pd.DataFrame(scores_mmGN)
baseline_GN = baseline_allGN.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_GN = baseline_GN.reset_index()
baseline_GN.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_GN['data_source'] = baseline_GN.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_GN['score_type'] = baseline_GN['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_GN['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_GN['score_order'] = baseline_GN['score_type'].map(score_type_ordermapD)
baseline_GN.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_GN['feature_group'] = feature_gp_nameGN
baseline_GN['sampling_type'] = sampling_type_name
baseline_GN['tts_split'] = tts_split_name
baseline_GN['n_features'] = n_featuresGN
###############################################################################
#all_featuresN = X_evolFN + X_structural_FN + X_genomicFN
# X_structural_FN = X_stability_FN + X_affinityFN + X_resprop_FN
# X_resprop_FN = X_aaindex_Fnum + X_str_Fnum + X_aap_Fcat
#================
# Structural cols
# X_structural_FN
#================
feature_gp_nameSTR = 'structural'
n_featuresSTR = len(X_structural_FN)
scores_mmSTR = MultModelsCl(input_df = X[X_structural_FN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_structural_FN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allSTR = pd.DataFrame(scores_mmSTR)
baseline_STR = baseline_allSTR.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_STR = baseline_STR.reset_index()
baseline_STR.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_STR['data_source'] = baseline_STR.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_STR['score_type'] = baseline_STR['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_STR['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_STR['score_order'] = baseline_STR['score_type'].map(score_type_ordermapD)
baseline_STR.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_STR['feature_group'] = feature_gp_nameSTR
baseline_STR['sampling_type'] = sampling_type_name
baseline_STR['tts_split'] = tts_split_name
baseline_STR['n_features'] = n_featuresSTR
##############################################################################
#================
# Stability cols
# X_stability_FN
#================
feature_gp_nameSTB = 'stability'
n_featuresSTB = len(X_stability_FN)
scores_mmSTB = MultModelsCl(input_df = X[X_stability_FN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_stability_FN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allSTB = pd.DataFrame(scores_mmSTB)
baseline_STB = baseline_allSTB.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_STB = baseline_STB.reset_index()
baseline_STB.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_STB['data_source'] = baseline_STB.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_STB['score_type'] = baseline_STB['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_STB['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_STB['score_order'] = baseline_STB['score_type'].map(score_type_ordermapD)
baseline_STB.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_STB['feature_group'] = feature_gp_nameSTB
baseline_STB['sampling_type'] = sampling_type_name
baseline_STB['tts_split'] = tts_split_name
baseline_STB['n_features'] = n_featuresSTB
###############################################################################
#================
# Affinity cols
# X_affinityFN
#================
feature_gp_nameAFF = 'affinity'
n_featuresAFF = len(X_affinityFN)
scores_mmAFF = MultModelsCl(input_df = X[X_affinityFN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_affinityFN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allAFF = pd.DataFrame(scores_mmAFF)
baseline_AFF = baseline_allAFF.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_AFF = baseline_AFF.reset_index()
baseline_AFF.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_AFF['data_source'] = baseline_AFF.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_AFF['score_type'] = baseline_AFF['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_AFF['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_AFF['score_order'] = baseline_AFF['score_type'].map(score_type_ordermapD)
baseline_AFF.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_AFF['feature_group'] = feature_gp_nameAFF
baseline_AFF['sampling_type'] = sampling_type_name
baseline_AFF['tts_split'] = tts_split_name
baseline_AFF['n_features'] = n_featuresAFF
###############################################################################
#================
# Residue level
# X_resprop_FN
#================
feature_gp_nameRES = 'residue_prop'
n_featuresRES = len(X_resprop_FN)
scores_mmRES = MultModelsCl(input_df = X[X_resprop_FN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_resprop_FN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allRES = pd.DataFrame(scores_mmRES)
baseline_RES = baseline_allRES.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_RES = baseline_RES.reset_index()
baseline_RES.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_RES['data_source'] = baseline_RES.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_RES['score_type'] = baseline_RES['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_RES['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_RES['score_order'] = baseline_RES['score_type'].map(score_type_ordermapD)
baseline_RES.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_RES['feature_group'] = feature_gp_nameRES
baseline_RES['sampling_type'] = sampling_type_name
baseline_RES['tts_split'] = tts_split_name
baseline_RES['n_features'] = n_featuresRES
###############################################################################
#================
# Residue level-AAindex
#X_resprop_FN - X_aaindex_Fnum
#================
X_respropNOaaFN = list(set(X_resprop_FN) - set(X_aaindex_Fnum))
feature_gp_nameRNAA = 'ResPropNoAA'
n_featuresRNAA = len(X_respropNOaaFN)
scores_mmRNAA = MultModelsCl(input_df = X[X_respropNOaaFN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_respropNOaaFN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allRNAA = pd.DataFrame(scores_mmRNAA)
baseline_RNAA = baseline_allRNAA.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_RNAA = baseline_RNAA.reset_index()
baseline_RNAA.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_RNAA['data_source'] = baseline_RNAA.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_RNAA['score_type'] = baseline_RNAA['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_RNAA['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_RNAA['score_order'] = baseline_RNAA['score_type'].map(score_type_ordermapD)
baseline_RNAA.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_RNAA['feature_group'] = feature_gp_nameRNAA
baseline_RNAA['sampling_type'] = sampling_type_name
baseline_RNAA['tts_split'] = tts_split_name
baseline_RNAA['n_features'] = n_featuresRNAA
###############################################################################
#================
# Structural cols-AAindex
#X_structural_FN - X_aaindex_Fnum
#================
X_strNOaaFN = list(set(X_structural_FN) - set(X_aaindex_Fnum))
feature_gp_nameSNAA = 'StrNoAA'
n_featuresSNAA = len(X_strNOaaFN)
scores_mmSNAA = MultModelsCl(input_df = X[X_strNOaaFN]
, target = y
, var_type = 'mixed'
, skf_cv = skf_cv
, blind_test_input_df = X_bts[X_strNOaaFN]
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
baseline_allSNAA = pd.DataFrame(scores_mmSNAA)
baseline_SNAA = baseline_allSNAA.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
baseline_SNAA = baseline_SNAA.reset_index()
baseline_SNAA.rename(columns = {'index': 'original_names'}, inplace = True)
# Indicate whether BT or CT
bt_pattern = re.compile(r'bts_.*')
baseline_SNAA['data_source'] = baseline_SNAA.apply(lambda row: 'BT' if bt_pattern.search(row.original_names) else 'CV' , axis = 1)
baseline_SNAA['score_type'] = baseline_SNAA['original_names'].str.replace('bts_|test_', '', regex = True)
score_type_uniqueN = set(baseline_SNAA['score_type'])
cL1 = list(score_type_ordermapD.keys())
cL2 = list(score_type_uniqueN)
if set(cL1).issubset(cL2):
print('\nPASS: sorting df by score that is mapped onto the order I want')
baseline_SNAA['score_order'] = baseline_SNAA['score_type'].map(score_type_ordermapD)
baseline_SNAA.sort_values(by = ['data_source', 'score_order'], ascending = [True, True], inplace = True)
else:
sys.exit('\nFAIL: could not sort df as score mapping for ordering failed')
baseline_SNAA['feature_group'] = feature_gp_nameSNAA
baseline_SNAA['sampling_type'] = sampling_type_name
baseline_SNAA['tts_split'] = tts_split_name
baseline_SNAA['n_features'] = n_featuresSNAA
###############################################################################
#%% COMBINING all FG dfs
#================
# Combine all
# https://stackoverflow.com/questions/39862654/pandas-concat-of-multiple-data-frames-using-only-common-columns
#================
dfs_combine = [baseline_EV, baseline_GN, baseline_STR, baseline_STB, baseline_AFF, baseline_RES , baseline_RNAA , baseline_SNAA]
dfs_nrows = []
for df in dfs_combine:
dfs_nrows = dfs_nrows + [len(df)]
dfs_nrows = max(dfs_nrows)
dfs_ncols = []
for df in dfs_combine:
dfs_ncols = dfs_ncols + [len(df.columns)]
dfs_ncols = max(dfs_ncols)
# dfs_ncols = []
# dfs_ncols2 = mode(dfs_ncols.append(len(df.columns) for df in dfs_combine)
# dfs_ncols2
expected_nrows = len(dfs_combine) * dfs_nrows
expected_ncols = dfs_ncols
common_cols = list(set.intersection(*(set(df.columns) for df in dfs_combine)))
if len(common_cols) == dfs_ncols :
combined_FG_baseline = pd.concat([df[common_cols] for df in dfs_combine], ignore_index=True)
fgs = combined_FG_baseline[['feature_group', 'n_features']]
fgs = fgs.drop_duplicates()
print('\nConcatenating dfs with feature groups after ML analysis (sampling type):'
, '\nNo. of dfs combining:', len(dfs_combine)
, '\nSampling type:', sampling_type
, '\nThe feature groups are:'
, '\n', fgs)
if len(combined_FG_baseline) == expected_nrows and len(combined_FG_baseline.columns) == expected_ncols:
print('\nPASS:', len(dfs_combine), 'dfs successfully combined'
, '\nnrows in combined_df:', len(combined_FG_baseline)
, '\nncols in combined_df:', len(combined_FG_baseline.columns))
else:
print('\nFAIL: concatenating failed'
, '\nExpected nrows:', expected_nrows
, '\nGot:', len(combined_FG_baseline)
, '\nExpected ncols:', expected_ncols
, '\nGot:', len(combined_FG_baseline.columns))
sys.exit()
else:
sys.exit('\nConcatenting dfs not possible,check numbers ')
# # rpow bind
# if all(ll((baseline_EV.columns == baseline_GN.columns == baseline_STR.columns)):
# print('\nPASS:colnames match, proceeding to rowbind')
# comb_df = pd.concat()], axis = 0, ignore_index = True )
###############################################################################
#====================
# Write output file
#====================
combined_FG_baseline.to_csv(outFile)
print('\nFile successfully written:', outFile)
###############################################################################

49
scripts/ml/ProcessMultModelsCl.py
View file

@ -5,9 +5,27 @@ Created on Thu Jun 23 20:39:20 2022
@author: tanu
"""
import os, sys
import pandas as pd
import numpy as np
import re
##############################################################################
#%% FUNCTION: Process outout dicr from MultModelsCl
def ProcessMultModelsCl(inputD = {}):
def ProcessMultModelCl(inputD = {}):
scoresDF = pd.DataFrame(inputD)
#------------------------
# Extracting split_name
#-----------------------
tts_split_nameL = []
for k,v in inputD.items():
tts_split_nameL = tts_split_nameL + [v['tts_split']]
if len(set(tts_split_nameL)) == 1:
tts_split_name = str(list(set(tts_split_nameL))[0])
print('\nExtracting tts_split_name:', tts_split_name)
#------------------------
# WF: only CV and BTS
#-----------------------
@ -28,7 +46,7 @@ def ProcessMultModelCl(inputD = {}):
#baseline_all = baseline_all_scores.filter(regex = 'bts_.*|test_.*|.*_time|TN|FP|FN|TP|.*_neg|.*_pos', axis = 0)
metaDF = scoresDFT.filter(regex='training_size|testSize|_time|TN|FP|FN|TP|.*_neg|.*_pos|resampling', axis = 1); scoresDF_BT.columns
metaDF = scoresDFT.filter(regex='training_size|blind_test_size|_time|TN|FP|FN|TP|.*_neg|.*_pos|resampling', axis = 1); scoresDF_BT.columns
#-----------------
# Combine WF
@ -38,8 +56,10 @@ def ProcessMultModelCl(inputD = {}):
print(scoresDF_CV)
print(scoresDF_BT)
print('\nCV dim:', scoresDF_CV.shape
, '\nBT dim:',scoresDF_BT.shape)
print('\nCombinig', len(dfs_combine_wf), 'using pd.concat by row ~ rowbind'
, '\nChecking Dims of df to combine:'
, '\nDim of CV:', scoresDF_CV.shape
, '\nDim of BT:', scoresDF_BT.shape)
dfs_nrows_wf = []
@ -57,14 +77,15 @@ def ProcessMultModelCl(inputD = {}):
expected_ncols_wf = dfs_ncols_wf
common_cols_wf = list(set.intersection(*(set(df.columns) for df in dfs_combine_wf)))
print('\nCOMMON COLS:', common_cols_wf
print('\nFinding Common cols to ensure row bind is correct:', len(common_cols_wf)
, '\nCOMMON cols are:', common_cols_wf
, dfs_ncols_wf)
if len(common_cols_wf) == dfs_ncols_wf :
combined_baseline_wf = pd.concat([df[common_cols_wf] for df in dfs_combine_wf], ignore_index=False)
#resampling_methods_wf = combined_baseline_wf[['resampling']]
#resampling_methods_wf = resampling_methods_wf.drop_duplicates()
print('\nConcatenating dfs with different resampling methods [WF]:', tts_split
print('\nConcatenating dfs with different resampling methods [WF]:', tts_split_name
, '\nNo. of dfs combining:', len(dfs_combine_wf))
print('\n================================================^^^^^^^^^^^^')
if len(combined_baseline_wf) == expected_nrows_wf and len(combined_baseline_wf.columns) == expected_ncols_wf:
@ -92,18 +113,4 @@ def ProcessMultModelCl(inputD = {}):
return combDF
# test
#ProcessMultModelCl(smnc_scores_mmD)
bazDF = MultModelsCl(input_df = X_smnc
, target = y_smnc
, var_type = 'mixed'
, tts_split_type = tts_split_7030
, resampling_type = 'smnc'
, skf_cv = skf_cv
, blind_test_df = X_bts
, blind_test_target = y_bts
, add_cm = True
, add_yn = True
, return_formatted_output = True)
###############################################################################

4
scripts/ml/run_7030_LOOP.py
View file

@ -175,7 +175,9 @@ smnc_scores_mmD = MultModelsCl(input_df = X_smnc
, blind_test_df = X_bts
, blind_test_target = y_bts
, add_cm = True
, add_yn = True)
, add_yn = True
, return_formatted_output = True):
)
smnc_all_scores = pd.DataFrame(smnc_scores_mmD)
rs_smnc = 'smnc'

5
scripts/ml/run_FS.py
View file

@ -248,8 +248,7 @@ with open(OutFileFS, 'w') as f:
# , cls = NpEncoder
))
# # read json
# with open(OutFileFS, 'r') as f:
# data = json.load(f)
# read json
with open(OutFileFS, 'r') as f:data = json.load(f)
##############################################################################