added cm_logo_skf.py and placeholder for splits

2022-07-01 13:55:12 +01:00 · 2022-07-01 13:55:12 +01:00 · d812835713
commit d812835713
parent 952cfeb4c0
4 changed files with 254 additions and 49 deletions
--- a/scripts/ml/combined_model/cm_logo_skf.py
+++ b/scripts/ml/combined_model/cm_logo_skf.py
@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Jun 29 19:44:06 2022
+
+@author: tanu
+"""
+import sys, os
+import pandas as pd
+import numpy as np
+import re
+###############################################################################
+homedir = os.path.expanduser("~")
+sys.path.append(homedir + '/git/LSHTM_analysis/scripts/ml/ml_functions')
+sys.path
+###############################################################################
+#====================
+# Import ML functions 
+#====================
+from ml_data_combined import *
+from MultClfs_logo_skf import *
+#from GetMLData import *
+#from SplitTTS import *
+
+skf_cv = StratifiedKFold(n_splits = 10 , shuffle = True,**rs)
+
+#logo = LeaveOneGroupOut()
+
+#%%
+def CMLogoSkf(combined_df
+        , all_genes =  ["embb", "katg", "rpob", "pnca", "gid", "alr"]
+        , bts_genes = ["embb", "katg", "rpob", "pnca", "gid"]
+        , cols_to_drop = ['dst', 'dst_mode', 'gene_name']
+        , target_var = 'dst_mode'
+        , gene_group = 'gene_name'
+        , std_gene_omit = []
+        ):
+
+    for bts_gene in bts_genes:
+        print('\n BTS gene:', bts_gene)
+            
+        tr_gene_omit = std_gene_omit + [bts_gene]
+        n_tr_genes = (len(bts_genes) - (len(std_gene_omit)))
+        #n_total_genes = (len(bts_genes) - len(std_gene_omit))
+        n_total_genes = len(all_genes)
+        
+        training_genesL = std_gene_omit + list(set(bts_genes) - set(tr_gene_omit))
+        #training_genesL = [element for element in bts_genes if element not in tr_gene_omit]
+    
+        print('\nTotal genes: ', n_total_genes
+              ,'\nTraining on:', n_tr_genes
+              ,'\nTraining on genes:', training_genesL
+              , '\nOmitted genes:', tr_gene_omit
+              , '\nBlind test gene:', bts_gene) 
+            
+        tts_split_type = "logoBT_" + bts_gene
+        
+        outFile = "/home/tanu/git/Data/ml_combined/" + str(n_tr_genes+1) + "genes_" + tts_split_type + ".csv"
+        print(outFile)
+    
+        #-------
+        # training
+        #------
+        cm_training_df = combined_df[~combined_df['gene_name'].isin(tr_gene_omit)]
+        
+        cm_X = cm_training_df.drop(cols_to_drop, axis=1, inplace=False)
+        #cm_y = cm_training_df.loc[:,'dst_mode']
+        cm_y = cm_training_df.loc[:, target_var]
+    
+    
+        gene_group = cm_training_df.loc[:,'gene_name']
+        
+        print('\nTraining data dim:', cm_X.shape
+              , '\nTraining Target dim:', cm_y.shape)
+        
+        if all(cm_X.columns.isin(cols_to_drop) == False):
+            print('\nChecked training df does NOT have Target var')
+        else:
+            sys.exit('\nFAIL: training data contains Target var')
+        
+        #---------------
+        # BTS: genes
+        #---------------
+        cm_test_df = combined_df[combined_df['gene_name'].isin([bts_gene])]
+        
+        cm_bts_X = cm_test_df.drop(cols_to_drop, axis = 1, inplace = False)
+        #cm_bts_y = cm_test_df.loc[:, 'dst_mode']
+        cm_bts_y = cm_test_df.loc[:, target_var]
+    
+        print('\nTraining data dim:', cm_bts_X.shape
+              , '\nTraining Target dim:', cm_bts_y.shape)
+        
+        
+        #%%:Running Multiple models on LOGO with SKF
+        cD3_v2 = MultModelsCl_logo_skf(input_df = cm_X
+                        , target = cm_y
+                        , group = 'none'
+                        , sel_cv = skf_cv
+        
+                        , blind_test_df = cm_bts_X
+                        , blind_test_target = cm_bts_y
+                        
+                        , tts_split_type = tts_split_type
+                        
+                        , resampling_type = 'none' # default
+                        , add_cm = True 
+                        , add_yn = True 
+                        , var_type = 'mixed'
+                        
+                        , run_blind_test = True
+                        , return_formatted_output = True
+                        , random_state = 42
+                        , n_jobs = 10
+                        )
+        
+        cD3_v2.to_csv(outFile)
+
+#%%
+CMLogoSkf(combined_df)
+CMLogoSkf(combined_df, std_gene_omit=['alr'])