.gitignore

@@ -13,3 +13,4 @@ scratch
 test
 plotting_test
 scripts_old
+foldx/test/

scripts/combining_dfs.py

@@ -54,6 +54,8 @@ os.getcwd()
 # FIXME: local imports
 #from combining import combine_dfs_with_checks
 from combining_FIXME import detect_common_cols
+from reference_dict import oneletter_aa_dict # CHECK DIR STRUC THERE!
+from reference_dict import low_3letter_dict # CHECK DIR STRUC THERE!
 #=======================================================================
 #%% command line args
 arg_parser = argparse.ArgumentParser()
@@ -71,13 +73,27 @@ args = arg_parser.parse_args()
 #%% variable assignment: input and output 
 #drug = 'pyrazinamide'
 #gene = 'pncA'
-gene_match = gene + '_p.'
 
 drug    = args.drug
 gene    = args.gene
 datadir = args.datadir
 indir   = args.input_dir
 outdir  = args.output_dir
+
+gene_match = gene + '_p.'
+print('mut pattern for gene', gene, ':',  gene_match)
+
+nssnp_match = gene_match +'[A-Za-z]{3}[0-9]+[A-Za-z]{3}'
+print('nsSNP for gene', gene, ':',  nssnp_match)
+
+wt_regex = gene_match.lower()+'([A-Za-z]{3})'
+print('wt regex:', wt_regex)
+
+mut_regex = r'[0-9]+(\w{3})$'
+print('mt regex:', mut_regex)
+
+pos_regex = r'([0-9]+)'
+print('position regex:', pos_regex)
 #%%=======================================================================
 #==============
 # directories
@@ -155,6 +171,8 @@ ncols_m1 = len(mcsm_foldx_dfs.columns)
 
 print('\n\nResult of first merge:', mcsm_foldx_dfs.shape
       , '\n===================================================================')
+mcsm_foldx_dfs[merging_cols_m1].apply(len)
+mcsm_foldx_dfs[merging_cols_m1].apply(len) == len(mcsm_foldx_dfs)
 #%%============================================================================
 print('==================================='
       , '\nSecond merge: dssp + kd'
@@ -183,6 +201,8 @@ ncols_m3 = len(dssp_kd_rd_dfs.columns)
 
 print('\n\nResult of Third merge:', dssp_kd_rd_dfs.shape
       , '\n===================================================================')
+dssp_kd_rd_dfs[merging_cols_m3].apply(len)
+dssp_kd_rd_dfs[merging_cols_m3].apply(len) == len(dssp_kd_rd_dfs)
 #%%============================================================================
 print('======================================='
       , '\nFourth merge: First merge + Third merge'
@@ -203,12 +223,16 @@ else:
 
 print('\nResult of Fourth merge:', combined_df.shape
       , '\n===================================================================')
+combined_df[merging_cols_m4].apply(len)
+combined_df[merging_cols_m4].apply(len) == len(combined_df)
 #%%============================================================================
 
 # OR merges: TEDIOUSSSS!!!!
 
-
+del(mcsm_df, foldx_df, mcsm_foldx_dfs, dssp_kd_dfs, dssp_kd_rd_dfs,rd_df, kd_df, infile_mcsm, infile_foldx, infile_dssp, infile_kd)
-#%%RRRR
+del(merging_cols_m1, merging_cols_m2, merging_cols_m3, merging_cols_m4)
+del(in_filename_dssp, in_filename_foldx, in_filename_kd, in_filename_mcsm, in_filename_rd)
+#%%
 print('==================================='
       , '\nFifth merge: afor_df + afor_kin_df'
       , '\n===================================')
@@ -220,8 +244,6 @@ afor_df = pd.read_csv(infile_afor, sep = ',')
 afor_kin_df = pd.read_csv(infile_afor_kin, sep = ',')
 afor_kin_df.columns = afor_kin_df.columns.str.lower()
 
-
-
 merging_cols_m5 = detect_common_cols(afor_df, afor_kin_df)
 
 print('Dim of afor_df:', afor_df.shape
@@ -230,7 +252,7 @@ print('Dim of afor_df:', afor_df.shape
 # finding if ALL afor_kin_df muts are present in afor_df
 # i.e all kinship muts should be PRESENT in mycalcs_present
 if len(afor_kin_df[afor_kin_df['mutation'].isin(afor_df['mutation'])]) == afor_kin_df.shape[0]:
-    print('PASS: ALL or_kinship muts are present in my or list')
+    print('PASS: ALL', len(afor_kin_df), 'or_kinship muts are present in my or list')
 else:
     nf_muts = len(afor_kin_df[~afor_kin_df['mutation'].isin(afor_df['mutation'])])
     nf_muts_df = afor_kin_df[~afor_kin_df['mutation'].isin(afor_df['mutation'])]    
@@ -241,10 +263,10 @@ else:
 
 # Now checking how many afor_df muts are NOT present in afor_kin_df    
 common_muts = len(afor_df[afor_df['mutation'].isin(afor_kin_df['mutation'])])  
-extra_muts_myor =   afor_kin_df.shape[0] - common_muts
+extra_muts_myor = afor_kin_df.shape[0] - common_muts
 
 print('=========================================='
-      , '\nmy or calcs', extra_muts_myor, 'extra mutation\n'
+      , '\nmy or calcs has', common_muts, 'present in af_or_kin_df'
       , '\n==========================================')
 
 print('Expected cals for merging with outer_join...')
@@ -252,23 +274,29 @@ print('Expected cals for merging with outer_join...')
 expected_rows = afor_df.shape[0] + extra_muts_myor
 expected_cols = afor_df.shape[1] + afor_kin_df.shape[1] - len(merging_cols_m5)
 
+
+afor_df['mutation']
+afor_kin_df['mutation']
+
 ors_df = pd.merge(afor_df, afor_kin_df, on = merging_cols_m5, how = o_join)
 
 if ors_df.shape[0] == expected_rows and ors_df.shape[1] == expected_cols:
-    print('PASS: OR dfs successfully combined! PHEWWWW!')
+    print('PASS but with duplicate muts: OR dfs successfully combined! PHEWWWW!'
+          , '\nDuplicate muts present but with different \'ref\' and \'alt\' alleles')
 else:
     print('FAIL: could not combine OR dfs'
           , '\nCheck expected rows and cols calculation and join type')
 
 print('Dim of merged ors_df:', ors_df.shape)
+
+ors_df[merging_cols_m5].apply(len)
+ors_df[merging_cols_m5].apply(len) == len(ors_df)
+
 #%%============================================================================
 # formatting ors_df
-
 ors_df.columns
 
-
 # Dropping unncessary columns: already removed in ealier preprocessing
-#cols_to_drop = ['reference_allele', 'alternate_allele', 'symbol' ]
 cols_to_drop = ['n_miss']
 print('Dropping', len(cols_to_drop), 'columns:\n'
       , cols_to_drop)
@@ -283,18 +311,13 @@ column_order = ['mutation'
                 , 'wild_type'               
                 , 'position'
                 , 'mutant_type'                              
-                #, 'chr_num_allele' #old
                 , 'ref_allele'
                 , 'alt_allele'
                 , 'mut_info_f1'
                 , 'mut_info_f2'
                 , 'mut_type'
                 , 'gene_id'
-                #, 'gene_number' #old
                 , 'gene_name'
-                #, 'mut_region'                
-                #, 'reference_allele'
-                #, 'alternate_allele'
                 , 'chromosome_number'                
                 , 'af'
                 , 'af_kin'               
@@ -317,14 +340,9 @@ column_order = ['mutation'
                 , 'se_kin'                            
                 , 'zval_logistic'
                 , 'logl_h1_kin'
-                , 'l_remle_kin'
+                , 'l_remle_kin']
-                #, 'wt_3let' # old
-                #, 'mt_3let' # old
-                #, 'symbol'
-                #, 'n_miss'
-                ]
 
-if ( (len(column_order) == ors_df.shape[1]) and (DataFrame(column_order).isin(ors_df.columns).all().all()):
+if ( (len(column_order) == ors_df.shape[1]) and (DataFrame(column_order).isin(ors_df.columns).all().all())):
     print('PASS: Column order generated for all:', len(column_order), 'columns'
           , '\nColumn names match, safe to reorder columns'
           , '\nApplying column order to df...' )
@@ -337,6 +355,61 @@ else:
 
 print('\nResult of Sixth merge:', ors_df_ordered.shape
       , '\n===================================================================')
+#%%
+ors_df_ordered.shape
+check = ors_df_ordered[['mutationinformation','mutation', 'wild_type', 'position', 'mutant_type']]
+
+# populating 'nan' info
+lookup_dict = dict()
+for k, v in low_3letter_dict.items():
+    lookup_dict[k] = v['one_letter_code']
+    #print(lookup_dict)
+    
+    wt = ors_df_ordered['mutation'].str.extract(wt_regex).squeeze()
+    #print(wt)
+    ors_df_ordered['wild_type'] = wt.map(lookup_dict)
+    
+    ors_df_ordered['position'] = ors_df_ordered['mutation'].str.extract(pos_regex) 
+    
+    mt = ors_df_ordered['mutation'].str.extract(mut_regex).squeeze()
+    ors_df_ordered['mutant_type'] = mt.map(lookup_dict)
+
+ors_df_ordered['mutationinformation'] = ors_df_ordered['wild_type'] + ors_df_ordered.position.map(str) + ors_df_ordered['mutant_type']
+check = ors_df_ordered[['mutationinformation','mutation', 'wild_type', 'position', 'mutant_type']]
+
+# populate mut_info_f1
+ors_df_ordered['mut_info_f1'].isna().sum()
+ors_df_ordered['mut_info_f1'] = ors_df_ordered['position'].astype(str) + ors_df_ordered['wild_type'] + '>' + ors_df_ordered['position'].astype(str) + ors_df_ordered['mutant_type']
+ors_df_ordered['mut_info_f1'].isna().sum()  
+
+# populate mut_info_f2
+ors_df_ordered['mut_info_f2'] = ors_df_ordered['mutation'].str.replace(gene_match.lower(), 'p.', regex = True)
+
+# populate mut_type
+ors_df_ordered['mut_type'].isna().sum()
+#mut_type_word = ors_df_ordered['mut_type'].value_counts()
+mut_type_word  = 'missense' # FIXME, should be derived
+ors_df_ordered['mut_type'].fillna(mut_type_word, inplace = True)
+ors_df_ordered['mut_type'].isna().sum()
+
+# populate gene_id
+ors_df_ordered['gene_id'].isna().sum()
+#gene_id_word = ors_df_ordered['gene_id'].value_counts()
+gene_id_word  = 'Rv2043c' # FIXME, should be derived
+ors_df_ordered['gene_id'].fillna(gene_id_word, inplace = True)
+ors_df_ordered['gene_id'].isna().sum()
+
+# populate gene_name
+ors_df_ordered['gene_name'].isna().sum()
+ors_df_ordered['gene_name'].value_counts()
+ors_df_ordered['gene_name'].fillna(gene, inplace = True)
+ors_df_ordered['gene_name'].isna().sum()
+
+# check numbers
+ors_df_ordered['or_kin'].isna().sum()
+# should be 0
+ors_df_ordered['or_mychisq'].isna().sum()
+
 #%%============================================================================ 
 print('==================================='
       , '\nSixth merge: Fourth + Fifth merge'
@@ -344,53 +417,159 @@ print('==================================='
       , '\n===================================')
 
 #combined_df_all = combine_dfs_with_checks(combined_df, ors_df_ordered, my_join = i_join)
-merging_cols_m6 = detect_common_cols(combined_df, ors_df_ordered)
+merging_cols_m6 = detect_common_cols(combined_df, ors_df_ordered) 
+
+# dtype problems
+if len(merging_cols_m6) > 1 and 'position'in merging_cols_m6:
+    print('Removing \'position\' from merging_cols_m6 to make dtypes consistent'
+          , '\norig length of merging_cols_m6:', len(merging_cols_m6))
+    merging_cols_m6.remove('position')
+    print('\nlength after removing:', len(merging_cols_m6))
+
 print('Dim of df1:', combined_df.shape
       , '\nDim of df2:', ors_df_ordered.shape
       , '\nNo. of merging_cols:', len(merging_cols_m6))
 
 print('Checking mutations in the two dfs:'
-      , '\nmuts in df1 but NOT in df2:'
+      , '\nmuts in df1 present in df2:'
       , combined_df['mutationinformation'].isin(ors_df_ordered['mutationinformation']).sum()
-      , '\nmuts in df2 but NOT in df1:'
+      , '\nmuts in df2 present in df1:'
       , ors_df_ordered['mutationinformation'].isin(combined_df['mutationinformation']).sum())
 
-#print('\nNo. of common muts:', np.intersect1d(combined_df['mutationinformation'], ors_df_ordered['mutationinformation']) )
+#----------
-
+# merge 6
-#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+#----------
-combined_df_all = pd.merge(combined_df, ors_df, on = merging_cols_m6, how = l_join)
+combined_df_all = pd.merge(combined_df, ors_df_ordered, on = merging_cols_m6, how = o_join)
 combined_df_all.shape
 
-# FIXME: DIM
+# sanity check for merge 6
-# only with left join!
+outdf_expected_rows = len(combined_df) + extra_muts_myor
-outdf_expected_rows = len(combined_df)
+unique_muts = len(combined_df)
 outdf_expected_cols = len(combined_df.columns) + len(ors_df_ordered.columns) - len(merging_cols_m6)
 
-#if combined_df_all.shape[1] == outdf_expected_cols and combined_df_all.shape[0] == outdf_expected_rows:  
+if combined_df_all.shape[0] == outdf_expected_rows and combined_df_all.shape[1] == outdf_expected_cols and combined_df_all['mutationinformation'].nunique() == unique_muts:
-if combined_df_all.shape[1] == outdf_expected_cols and combined_df_all['mutationinformation'].nunique() == outdf_expected_rows:
     print('PASS: Df dimension match'
-          , '\nDim of combined_df_all with join type:', l_join
+          , '\ncombined_df_all with join type:', o_join
           , '\n', combined_df_all.shape
           , '\n===============================================================')
 else:
     print('FAIL: Df dimension mismatch'
              , 'Cannot generate expected dim. See details of merge performed'
              , '\ndf1 dim:', combined_df.shape
-             , '\ndf2 dim:', ors_df.shape
+             , '\ndf2 dim:', ors_df_ordered.shape
              , '\nGot:', combined_df_all.shape
              , '\nmuts in df1 but NOT in df2:'
-             , combined_df['mutationinformation'].isin(ors_df['mutationinformation']).sum()
+             , combined_df['mutationinformation'].isin(ors_df_ordered['mutationinformation']).sum()
              , '\nmuts in df2 but NOT in df1:'
              , ors_df['mutationinformation'].isin(combined_df['mutationinformation']).sum())
     sys.exit()
-#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ 
-# nan in mutation col
+# drop extra cols
-# FIXME: should get fixmed with JP's resolved dataset!?
+all_cols = combined_df_all.columns
-combined_df_all['mutation'].isna().sum()
+
-baz = combined_df_all[combined_df_all['mutation'].isna()]
+#pos_cols_check = combined_df_all[['position_x','position_y']]
+c = combined_df_all[['position_x','position_y']].isna().sum()
+pos_col_to_drop = c.index[c>0].to_list()
+cols_to_drop = pos_col_to_drop +  ['wild_type_kd']   
+
+print('Dropping', len(cols_to_drop), 'columns:\n', cols_to_drop)
+combined_df_all.drop(cols_to_drop, axis = 1, inplace = True) 
+ 
+# rename position_x to position
+pos_col_to_rename = c.index[c==0].to_list()
+combined_df_all.shape
+combined_df_all.rename(columns = { pos_col_to_rename[0]: 'position'}, inplace = True)
+combined_df_all.shape
+
+all_cols = combined_df_all.columns
+
+#%% reorder cols to for convenience
+first_cols = ['mutationinformation','mutation', 'wild_type', 'position', 'mutant_type']
+last_cols = [col for col in combined_df_all.columns if col not in first_cols]
+
+combined_df_all = combined_df_all[first_cols+last_cols]
+
+#%% IMPORTANT: check if mutation related info is all populated after this merge  
+# select string colnames to ensure no NA exist there
+string_cols = combined_df_all.columns[combined_df_all.applymap(lambda x: isinstance(x, str)).all(0)]
+
+if (combined_df_all[string_cols].isna().sum(axis = 0)== 0).all():
+    print('PASS: All string cols are populated with no NAs')
+else:
+    print('FAIL: NAs detected in string cols')
+    print(combined_df_all[string_cols].isna().sum(axis = 0))
+    sys.exit()
+
+# relevant mut cols
+check_mut_cols = merging_cols_m5 + merging_cols_m6
+
+count_na_mut_cols = combined_df_all[check_mut_cols].isna().sum().reset_index().rename(columns = {'index': 'col_name', 0: 'na_count'})
+print(check_mut_cols)
+
+c2 = combined_df_all[check_mut_cols].isna().sum()
+missing_info_cols = c2.index[c2>0].to_list()
+
+if c2.sum()>0:
+    #na_muts_n = combined_df_all['mutation'].isna().sum() 
+    na_muts_n = combined_df_all[missing_info_cols].isna().sum() 
+    print(na_muts_n.values[0], 'mutations have missing \'mutation\' info.'
+          , '\nFetching these from reference dict...')
+else:
+    print('No missing \'mutation\' has been detected!')
+    
+lookup_dict = dict()
+for k, v in oneletter_aa_dict.items():
+    lookup_dict[k] = v['three_letter_code_lower']
+    print(lookup_dict)
+    wt_3let = combined_df_all['wild_type'].map(lookup_dict)
+    #print(wt_3let)
+    pos = combined_df_all['position'].astype(str)
+    #print(pos)
+    mt_3let = combined_df_all['mutant_type'].map(lookup_dict)
+    #print(mt_3let)
+    # override the 'mutation' column
+    combined_df_all['mutation'] = 'pnca_p.' + wt_3let + pos + mt_3let
+    print(combined_df_all['mutation'])    
+
+# check again
+if combined_df_all[missing_info_cols].isna().sum().all() == 0: 
+    print('PASS: No mutations have missing \'mutation\' info.')
+else:
+    print('FAIL:', combined_df_all[missing_info_cols].isna().sum().values[0]
+          , '\nmutations have missing info STILL...')
+    sys.exit()
+
+#%% check
+foo = combined_df_all.drop_duplicates('mutationinformation')
+foo2 = combined_df_all.drop_duplicates('mutation')
+if foo.equals(foo2):
+    print('PASS: Dropping mutation or mutatationinformation has the same effect\n')
+else:
+    print('FAIL: Still problems in merged data')
+    sys.exit()
+
 #%%============================================================================
 output_cols = combined_df_all.columns
-print('Output cols:', output_cols)
+
+#%% IMPORTANT result info
+if combined_df_all['or_mychisq'].isna().sum() == len(combined_df) - len(afor_df):
+    print('PASS: No. of NA in or_mychisq matches expected length'
+          , '\nNo. of with NA in or_mychisq:', combined_df_all['or_mychisq'].isna().sum() 
+          , '\nNo. of NA in or_kin:', combined_df_all['or_kin'].isna().sum())
+else:
+    print('FAIL: No. of NA in or_mychisq does not match expected length')
+
+
+if combined_df_all.shape[0] == outdf_expected_rows:
+    print('\nINFORMARIONAL ONLY: combined_df_all has duplicate muts present but with unique ref and alt allele'
+          , '\n=============================================================')
+else:
+    print('combined_df_all has no duplicate muts present'
+          ,'\n===============================================================')
+    
+print('\nDim of combined_data:', combined_df_all.shape
+      , '\nNo. of unique mutations:', combined_df_all['mutationinformation'].nunique())
+    
 
 #%%============================================================================ 
 # write csv

scripts/data_extraction.py

@@ -45,8 +45,6 @@ Created on Tue Aug  6 12:56:03 2019
 #5. chain
 #6. wild_pos
 #7. wild_chain_pos
-
-
 #=======================================================================
 #%% load libraries
 import os, sys
@@ -83,16 +81,16 @@ gene = args.gene
 gene_match = gene + '_p.'
 print('mut pattern for gene', gene, ':',  gene_match)
 
-nssnp_match = gene_match +'[A-Z]{3}[0-9]+[A-Z]{3}'
+nssnp_match = gene_match +'[A-Za-z]{3}[0-9]+[A-Za-z]{3}'
 print('nsSNP for gene', gene, ':',  nssnp_match)
 
-wt_regex = gene_match.lower()+'(\w{3})'
+wt_regex = gene_match.lower()+'([A-Za-z]{3})'
 print('wt regex:', wt_regex)
 
-mut_regex = r'\d+(\w{3})$'
+mut_regex = r'[0-9]+(\w{3})$'
 print('mt regex:', mut_regex)
 
-pos_regex = r'(\d+)'
+pos_regex = r'([0-9]+)'
 print('position regex:', pos_regex)
 
 # building cols to extract
@@ -156,30 +154,29 @@ if in_filename_master == 'original_tanushree_data_v2.csv':
 else:
     core_cols = ['id'
                  , 'sample'
-                 , 'patient_id'
+                 #, 'patient_id'
-                 , 'strain'
+                 #, 'strain'
                  , 'lineage'
                  , 'sublineage' 
-                 , 'country'
+                 #, 'country'
                  , 'country_code'
                  , 'geographic_source'
                  #, 'region'
-                 , 'location'
+                 #, 'location'
-                 , 'host_body_site'
+                 #, 'host_body_site'
-                 , 'environment_material'
+                 #, 'environment_material'
-                 , 'host_status'
+                 #, 'host_status'
-                 , 'host_sex'
+                 #, 'host_sex'
-                 , 'submitted_host_sex'
+                 #, 'submitted_host_sex'
-                 , 'hiv_status'
+                 #, 'hiv_status'
-                 , 'HIV_status'
+                 #, 'HIV_status'
-                 , 'tissue_type'
+                 #, 'tissue_type'
-                 , 'isolation_source'
+                 #, 'isolation_source'
                  ,  resistance_col]
     
     variable_based_cols = [drug
                            , dr_muts_col
                            , other_muts_col]
-                           #, resistance_col]
        
     cols_to_extract = core_cols + variable_based_cols
     print('Extracting', len(cols_to_extract), 'columns from master data')
@@ -202,7 +199,7 @@ print('No. of NAs/column:' + '\n', meta_data.isna().sum()
 
 #%% Write check file
 check_file = outdir + '/' + gene.lower() + '_gwas.csv'
-meta_data.to_csv(check_file)
+meta_data.to_csv(check_file, index = False)
 print('Writing subsetted gwas data'
       , '\nFile', check_file
       , '\nDim:', meta_data.shape)
@@ -217,9 +214,9 @@ print('Writing subsetted gwas data'
 # drug counts: complete samples for OR calcs
 meta_data[drug].value_counts() 
 print('===========================================================\n'
-      , 'RESULT: No. of Sus and Res samples:\n', meta_data[drug].value_counts()
+      , 'RESULT: No. of Sus and Res', drug, 'samples:\n', meta_data[drug].value_counts()
 	  , '\n===========================================================\n'
-      , 'RESULT: Percentage of Sus and Res samples:\n', meta_data[drug].value_counts(normalize = True)*100
+      , 'RESULT: Percentage of Sus and Res', drug, 'samples:\n', meta_data[drug].value_counts(normalize = True)*100
 		, '\n===========================================================')
 
 #%%
@@ -1173,7 +1170,7 @@ del(out_filename_metadata_poscounts)
 
 #%% Write file: gene_metadata (i.e gene_LF1)
 # where each row has UNIQUE mutations NOT unique sample ids
-out_filename_metadata = gene.lower() + '_metadata.csv'
+out_filename_metadata = gene.lower() + '_metadata_raw.csv'
 outfile_metadata = outdir + '/' + out_filename_metadata
 print('Writing file: LF formatted data'
       , '\nFile:', outfile_metadata

scripts/ks_test_PS.R

@@ -0,0 +1,211 @@
+#!/usr/bin/env Rscript
+#########################################################
+# TASK: KS test for PS/DUET lineage distributions
+#=======================================================================
+#=======================================================================
+# working dir and loading libraries
+getwd()
+setwd("~/git/LSHTM_analysis/scripts/")
+getwd()
+
+#source("/plotting/Header_TT.R")
+#source("../barplot_colour_function.R")
+#require(data.table)
+source("plotting/combining_dfs_plotting.R")
+# should return the following dfs, directories and variables
+
+# PS combined: 
+# 1) merged_df2
+# 2) merged_df2_comp
+# 3) merged_df3
+# 4) merged_df3_comp
+
+# LIG combined: 
+# 5) merged_df2_lig
+# 6) merged_df2_comp_lig
+# 7) merged_df3_lig
+# 8) merged_df3_comp_lig
+
+# 9) my_df_u
+# 10) my_df_u_lig
+
+cat(paste0("Directories imported:"
+           , "\ndatadir:", datadir
+           , "\nindir:", indir
+           , "\noutdir:", outdir
+           , "\nplotdir:", plotdir))
+
+cat(paste0("Variables imported:"
+           , "\ndrug:", drug
+           , "\ngene:", gene
+           , "\ngene_match:", gene_match
+           , "\nAngstrom symbol:", angstroms_symbol
+           , "\nNo. of duplicated muts:", dup_muts_nu
+           , "\nNA count for ORs:", na_count
+           , "\nNA count in df2:", na_count_df2
+           , "\nNA count in df3:", na_count_df3))       
+
+###########################
+# Data for  stats
+# you need merged_df2 or merged_df2_comp
+# since this is one-many relationship 
+# i.e the same SNP can belong to multiple lineages
+# using the _comp dataset means
+# we lose some muts and at this level, we should use
+# as much info as available, hence use df with NA
+###########################
+
+# REASSIGNMENT
+my_df  = merged_df2
+
+# delete variables not required
+rm(merged_df2, merged_df2_comp, merged_df3, merged_df3_comp)
+
+# quick checks
+colnames(my_df)
+str(my_df)
+
+# Ensure correct data type in columns to plot: need to be factor
+is.factor(my_df$lineage)
+my_df$lineage = as.factor(my_df$lineage)
+is.factor(my_df$lineage)
+
+table(my_df$mutation_info); str(my_df$mutation_info)
+
+# subset df with dr muts only
+my_df_dr = subset(my_df, mutation_info == "dr_mutations_pyrazinamide") 
+table(my_df_dr$mutation_info)
+
+# stats for all muts and dr_muts
+# 1) for all muts
+# 2) for dr_muts
+#===========================
+table(my_df$lineage); str(my_df$lineage)
+table(my_df_dr$lineage); str(my_df_dr$lineage)
+
+# subset only lineages1-4
+sel_lineages = c("lineage1"
+                 , "lineage2"
+                 , "lineage3"
+                 , "lineage4")
+
+# subset and refactor: all muts
+df_lin = subset(my_df, subset = lineage %in% sel_lineages)
+df_lin$lineage = factor(df_lin$lineage)
+
+# subset and refactor: dr muts
+df_lin_dr = subset(my_df_dr, subset = lineage %in% sel_lineages)
+df_lin_dr$lineage = factor(df_lin_dr$lineage)
+
+
+#{RESULT: No of samples within lineage}
+table(df_lin$lineage) 
+table(df_lin_dr$lineage)
+
+#{Result: No. of unique mutations the 4 lineages contribute to}
+length(unique(df_lin$mutationinformation))
+length(unique(df_lin_dr$mutationinformation))
+
+
+# COMPARING DISTRIBUTIONS
+#================
+# ALL mutations
+#=================
+head(df_lin$lineage)
+df_lin$lineage = as.character(df_lin$lineage)
+
+lin1 = df_lin[df_lin$lineage == "lineage1",]$duet_scaled
+lin2 = df_lin[df_lin$lineage == "lineage2",]$duet_scaled
+lin3 = df_lin[df_lin$lineage == "lineage3",]$duet_scaled
+lin4 = df_lin[df_lin$lineage == "lineage4",]$duet_scaled
+
+# ks test
+lin12 = ks.test(lin1,lin2) 
+lin12_df = as.data.frame(cbind(lin12$data.name, lin12$p.value))
+  
+lin13 = ks.test(lin1,lin3) 
+lin13_df = as.data.frame(cbind(lin13$data.name, lin13$p.value))
+
+lin14 = ks.test(lin1,lin4) 
+lin14_df = as.data.frame(cbind(lin14$data.name, lin14$p.value))
+
+lin23 = ks.test(lin2,lin3)
+lin23_df = as.data.frame(cbind(lin23$data.name, lin23$p.value))
+
+lin24 = ks.test(lin2,lin4)  
+lin24_df = as.data.frame(cbind(lin24$data.name, lin24$p.value))
+
+lin34 = ks.test(lin3,lin4)
+lin34_df = as.data.frame(cbind(lin34$data.name, lin34$p.value))
+
+ks_results_all = rbind(lin12_df 
+                   , lin13_df
+                   , lin14_df
+                   , lin23_df
+                   , lin24_df
+                   , lin34_df)
+
+#p-value < 2.2e-16
+rm(lin12, lin12_df
+   , lin13, lin13_df
+   , lin14, lin14_df
+   , lin23, lin23_df
+   , lin24, lin24_df
+   , lin34, lin34_df)
+
+#================
+# DRUG mutations
+#=================
+head(df_lin_dr$lineage)
+df_lin_dr$lineage = as.character(df_lin_dr$lineage)
+
+lin1_dr = df_lin_dr[df_lin_dr$lineage == "lineage1",]$duet_scaled
+lin2_dr = df_lin_dr[df_lin_dr$lineage == "lineage2",]$duet_scaled
+lin3_dr = df_lin_dr[df_lin_dr$lineage == "lineage3",]$duet_scaled
+lin4_dr = df_lin_dr[df_lin_dr$lineage == "lineage4",]$duet_scaled
+
+# ks test: dr muts
+lin12_dr = ks.test(lin1_dr,lin2_dr) 
+lin12_df_dr = as.data.frame(cbind(lin12_dr$data.name, lin12_dr$p.value))
+
+lin13_dr = ks.test(lin1_dr,lin3_dr) 
+lin13_df_dr = as.data.frame(cbind(lin13_dr$data.name, lin13_dr$p.value))
+
+lin14_dr = ks.test(lin1_dr,lin4_dr) 
+lin14_df_dr = as.data.frame(cbind(lin14_dr$data.name, lin14_dr$p.value))
+
+lin23_dr = ks.test(lin2_dr,lin3_dr)
+lin23_df_dr = as.data.frame(cbind(lin23_dr$data.name, lin23_dr$p.value))
+
+lin24_dr = ks.test(lin2_dr,lin4_dr)  
+lin24_df_dr = as.data.frame(cbind(lin24_dr$data.name, lin24_dr$p.value))
+
+lin34_dr = ks.test(lin3_dr,lin4_dr)
+lin34_df_dr = as.data.frame(cbind(lin34_dr$data.name, lin34_dr$p.value))
+
+ks_results_dr = rbind(lin12_df_dr 
+                      , lin13_df_dr
+                      , lin14_df_dr
+                      , lin23_df_dr
+                      , lin24_df_dr
+                      , lin34_df_dr)
+
+ks_results_combined = cbind(ks_results_all, ks_results_dr)
+
+my_colnames = c("Lineage_comparisons"
+                , paste0("All_mutations n=", nrow(df_lin))
+                , paste0("Drug_associated_mutations n=", nrow(df_lin_dr)))
+my_colnames
+
+# select the output columns
+ks_results_combined_f = ks_results_combined[,c(1,2,4)]
+
+colnames(ks_results_combined_f) = my_colnames
+ks_results_combined_f 
+
+#=============
+# write output file
+#=============
+ks_results = paste0(outdir,"/results/ks_results.csv")
+write.csv(ks_results_combined_f, ks_results, row.names = F)
+

scripts/or_kinship_link.py

@@ -104,7 +104,7 @@ or_df.columns
 
 #%% snp_info file: master and gene specific ones
 # gene info
-info_df2  = pd.read_csv(gene_info, sep = '\t', header = 0) #447, 10 
+info_df2  = pd.read_csv(gene_info, sep = '\t', header = 0) #447, 11
 #info_df2  = pd.read_csv(gene_info, sep = ',', header = 0) #447 10   
 mis_mut_cover = (info_df2['chromosome_number'].nunique()/info_df2['chromosome_number'].count()) * 100
 print('*****RESULT*****'
@@ -212,7 +212,7 @@ else:
 
 #PENDING Jody's reply 
 # !!!!!!!! 
-# drop nan from dfm2_mis as these are not useful
+# drop nan from dfm2_mis as these are not useful and JP confirmed the same
 print('Dropping NAs before further processing...')
 dfm2_mis = dfm2[dfm2['mut_type'].notnull()]
 # !!!!!!!!

scripts/plotting/autoviz.py

@@ -1,45 +0,0 @@
-#!/usr/bin/env python3
-#=======================================================================
-#%% useful links
-#https://towardsdatascience.com/autoviz-automatically-visualize-any-dataset-ba2691a8b55a
-#https://pypi.org/project/autoviz/
-#=======================================================================
-import os, sys
-import pandas as pd
-import numpy as np
-import re
-import argparse
-from autoviz.AutoViz_Class import AutoViz_Class
-
-homedir = os.path.expanduser('~')
-os.chdir(homedir + '/git/LSHTM_analysis/scripts')
-#%%============================================================================
-# variables
-gene = 'pncA'
-drug = 'pyrazinamide'
-
-#%%============================================================================
-#==============
-# directories
-#==============
-datadir = homedir + '/' + 'git/Data'
-    
-indir = datadir + '/' + drug + '/input'
-    
-outdir = datadir + '/' + drug + '/output'
-
-#=======
-# input
-#=======
-in_filename_plotting = 'car_design.csv'
-in_filename_plotting = gene.lower() + '_all_params.csv'
-infile_plotting = outdir + '/' + in_filename_plotting
-print('plotting file: ', infile_plotting
-      , '\n============================================================')
-#=======================================================================
-plotting_df = pd.read_csv(infile_plotting, sep = ',')
-#Instantiate the AutoViz class
-AV = AutoViz_Class()
-df = AV.AutoViz(infile_plotting)
-#df2 = AV.AutoViz(plotting_df)
-plotting_df.columns[~plotting_df.columns.isin(df.columns)]

scripts/plotting/barplots_subcolours_aa_PS.R

@@ -1,5 +1,4 @@
-#!/usr/bin/env Rscript   
+#!/usr/bin/env Rscript   getwd()
-getwd()
 setwd("~/git/LSHTM_analysis/scripts/plotting")
 getwd()
 
@@ -19,8 +18,7 @@ getwd()
 library(ggplot2)
 library(data.table)
 source("barplot_colour_function.R")
-#source("subcols_axis.R")
+source("subcols_axis.R")
-source("subcols_axis_PS.R")
 
 # should return the following dfs, directories and variables
 # mut_pos_cols
@@ -161,9 +159,7 @@ min(df$duet_scaled)
 max(df$duet_scaled)
 
 # sanity checks
-# very important!!!!
 tapply(df$duet_scaled, df$duet_outcome, min)
-
 tapply(df$duet_scaled, df$duet_outcome, max)
 
 # My colour FUNCTION: based on group and subgroup
@@ -241,7 +237,7 @@ outPlot = g +
          , axis.ticks.x = element_blank()) +
   labs(title = ""
        #title = my_title
-       , x = "position"
+       , x = "Position"
        , y = "Frequency")
 
 print(outPlot)

scripts/plotting/columns_all_params.csv

@@ -1,87 +0,0 @@
-,x,,changes,
-1,mutationinformation,,Mutationinformation,
-2,wild_type,,,consider...wild_aa
-3,position,,Position,
-4,mutant_type,,,consider...mutant_aa
-5,chain,,,
-6,ligand_id,,,
-7,ligand_distance,,,
-8,duet_stability_change,,,
-9,duet_outcome,,DUET_outcome,
-10,ligand_affinity_change,,,
-11,ligand_outcome,,Lig_outcome,
-12,duet_scaled,,ratioDUET,
-13,affinity_scaled,,ratioPredAff,
-14,wild_pos,,WildPos,
-15,wild_chain_pos,,,
-16,ddg,,,
-17,contacts,,,
-18,electro_rr,,,
-19,electro_mm,,,
-20,electro_sm,,,
-21,electro_ss,,,
-22,disulfide_rr,,,
-23,disulfide_mm,,,
-24,disulfide_sm,,,
-25,disulfide_ss,,,
-26,hbonds_rr,,,
-27,hbonds_mm,,,
-28,hbonds_sm,,,
-29,hbonds_ss,,,
-30,partcov_rr,,,
-31,partcov_mm,,,
-32,partcov_sm,,,
-33,partcov_ss,,,
-34,vdwclashes_rr,,,
-35,vdwclashes_mm,,,
-36,vdwclashes_sm,,,
-37,vdwclashes_ss,,,
-38,volumetric_rr,,,
-39,volumetric_mm,,,
-40,volumetric_sm,,,
-41,volumetric_ss,,,
-42,wild_type_dssp,,,
-43,asa,,,
-44,rsa,,,
-45,ss,,,
-46,ss_class,,,
-47,chain_id,,,
-48,wild_type_kd,,,
-49,kd_values,,,
-50,rd_values,,,
-51,wt_3letter_caps,,,
-52,mutation,,,
-53,af,,,
-54,beta_logistic,,,
-55,or_logistic,,,
-56,pval_logistic,,,
-57,se_logistic,,,
-58,zval_logistic,,,
-59,ci_low_logistic,,,
-60,ci_hi_logistic,,,
-61,or_mychisq,,,
-62,or_fisher,,,
-63,pval_fisher,,,
-64,ci_low_fisher,,,
-65,ci_hi_fisher,,,
-66,est_chisq,,,
-67,pval_chisq,,,
-68,chromosome_number,,,
-69,ref_allele,,,
-70,alt_allele,,,
-71,mut_type,,,
-72,gene_id,,,
-73,gene_number,,,
-74,mut_region,,,
-75,mut_info,,,
-76,chr_num_allele,,,
-77,wt_3let,,,
-78,mt_3let,,,
-79,af_kin,,,
-80,or_kin,,,
-81,pwald_kin,,,
-82,beta_kin,,,
-83,se_kin,,,
-84,logl_h1_kin,,,
-85,l_remle_kin,,,
-86,n_miss,,,