gidb_dev #1
21 changed files with 2132 additions and 243 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -13,3 +13,4 @@ scratch
|
|||
test
|
||||
plotting_test
|
||||
scripts_old
|
||||
foldx/test/
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ os.getcwd()
|
|||
# FIXME: local imports
|
||||
#from combining import combine_dfs_with_checks
|
||||
from combining_FIXME import detect_common_cols
|
||||
from reference_dict import oneletter_aa_dict # CHECK DIR STRUC THERE!
|
||||
from reference_dict import low_3letter_dict # CHECK DIR STRUC THERE!
|
||||
#=======================================================================
|
||||
#%% command line args
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
|
|
@ -71,13 +73,27 @@ args = arg_parser.parse_args()
|
|||
#%% variable assignment: input and output
|
||||
#drug = 'pyrazinamide'
|
||||
#gene = 'pncA'
|
||||
gene_match = gene + '_p.'
|
||||
|
||||
drug = args.drug
|
||||
gene = args.gene
|
||||
datadir = args.datadir
|
||||
indir = args.input_dir
|
||||
outdir = args.output_dir
|
||||
|
||||
gene_match = gene + '_p.'
|
||||
print('mut pattern for gene', gene, ':', gene_match)
|
||||
|
||||
nssnp_match = gene_match +'[A-Za-z]{3}[0-9]+[A-Za-z]{3}'
|
||||
print('nsSNP for gene', gene, ':', nssnp_match)
|
||||
|
||||
wt_regex = gene_match.lower()+'([A-Za-z]{3})'
|
||||
print('wt regex:', wt_regex)
|
||||
|
||||
mut_regex = r'[0-9]+(\w{3})$'
|
||||
print('mt regex:', mut_regex)
|
||||
|
||||
pos_regex = r'([0-9]+)'
|
||||
print('position regex:', pos_regex)
|
||||
#%%=======================================================================
|
||||
#==============
|
||||
# directories
|
||||
|
|
@ -155,6 +171,8 @@ ncols_m1 = len(mcsm_foldx_dfs.columns)
|
|||
|
||||
print('\n\nResult of first merge:', mcsm_foldx_dfs.shape
|
||||
, '\n===================================================================')
|
||||
mcsm_foldx_dfs[merging_cols_m1].apply(len)
|
||||
mcsm_foldx_dfs[merging_cols_m1].apply(len) == len(mcsm_foldx_dfs)
|
||||
#%%============================================================================
|
||||
print('==================================='
|
||||
, '\nSecond merge: dssp + kd'
|
||||
|
|
@ -183,6 +201,8 @@ ncols_m3 = len(dssp_kd_rd_dfs.columns)
|
|||
|
||||
print('\n\nResult of Third merge:', dssp_kd_rd_dfs.shape
|
||||
, '\n===================================================================')
|
||||
dssp_kd_rd_dfs[merging_cols_m3].apply(len)
|
||||
dssp_kd_rd_dfs[merging_cols_m3].apply(len) == len(dssp_kd_rd_dfs)
|
||||
#%%============================================================================
|
||||
print('======================================='
|
||||
, '\nFourth merge: First merge + Third merge'
|
||||
|
|
@ -203,12 +223,16 @@ else:
|
|||
|
||||
print('\nResult of Fourth merge:', combined_df.shape
|
||||
, '\n===================================================================')
|
||||
combined_df[merging_cols_m4].apply(len)
|
||||
combined_df[merging_cols_m4].apply(len) == len(combined_df)
|
||||
#%%============================================================================
|
||||
|
||||
# OR merges: TEDIOUSSSS!!!!
|
||||
|
||||
|
||||
#%%RRRR
|
||||
del(mcsm_df, foldx_df, mcsm_foldx_dfs, dssp_kd_dfs, dssp_kd_rd_dfs,rd_df, kd_df, infile_mcsm, infile_foldx, infile_dssp, infile_kd)
|
||||
del(merging_cols_m1, merging_cols_m2, merging_cols_m3, merging_cols_m4)
|
||||
del(in_filename_dssp, in_filename_foldx, in_filename_kd, in_filename_mcsm, in_filename_rd)
|
||||
#%%
|
||||
print('==================================='
|
||||
, '\nFifth merge: afor_df + afor_kin_df'
|
||||
, '\n===================================')
|
||||
|
|
@ -220,8 +244,6 @@ afor_df = pd.read_csv(infile_afor, sep = ',')
|
|||
afor_kin_df = pd.read_csv(infile_afor_kin, sep = ',')
|
||||
afor_kin_df.columns = afor_kin_df.columns.str.lower()
|
||||
|
||||
|
||||
|
||||
merging_cols_m5 = detect_common_cols(afor_df, afor_kin_df)
|
||||
|
||||
print('Dim of afor_df:', afor_df.shape
|
||||
|
|
@ -230,7 +252,7 @@ print('Dim of afor_df:', afor_df.shape
|
|||
# finding if ALL afor_kin_df muts are present in afor_df
|
||||
# i.e all kinship muts should be PRESENT in mycalcs_present
|
||||
if len(afor_kin_df[afor_kin_df['mutation'].isin(afor_df['mutation'])]) == afor_kin_df.shape[0]:
|
||||
print('PASS: ALL or_kinship muts are present in my or list')
|
||||
print('PASS: ALL', len(afor_kin_df), 'or_kinship muts are present in my or list')
|
||||
else:
|
||||
nf_muts = len(afor_kin_df[~afor_kin_df['mutation'].isin(afor_df['mutation'])])
|
||||
nf_muts_df = afor_kin_df[~afor_kin_df['mutation'].isin(afor_df['mutation'])]
|
||||
|
|
@ -241,10 +263,10 @@ else:
|
|||
|
||||
# Now checking how many afor_df muts are NOT present in afor_kin_df
|
||||
common_muts = len(afor_df[afor_df['mutation'].isin(afor_kin_df['mutation'])])
|
||||
extra_muts_myor = afor_kin_df.shape[0] - common_muts
|
||||
extra_muts_myor = afor_kin_df.shape[0] - common_muts
|
||||
|
||||
print('=========================================='
|
||||
, '\nmy or calcs', extra_muts_myor, 'extra mutation\n'
|
||||
, '\nmy or calcs has', common_muts, 'present in af_or_kin_df'
|
||||
, '\n==========================================')
|
||||
|
||||
print('Expected cals for merging with outer_join...')
|
||||
|
|
@ -252,23 +274,29 @@ print('Expected cals for merging with outer_join...')
|
|||
expected_rows = afor_df.shape[0] + extra_muts_myor
|
||||
expected_cols = afor_df.shape[1] + afor_kin_df.shape[1] - len(merging_cols_m5)
|
||||
|
||||
|
||||
afor_df['mutation']
|
||||
afor_kin_df['mutation']
|
||||
|
||||
ors_df = pd.merge(afor_df, afor_kin_df, on = merging_cols_m5, how = o_join)
|
||||
|
||||
if ors_df.shape[0] == expected_rows and ors_df.shape[1] == expected_cols:
|
||||
print('PASS: OR dfs successfully combined! PHEWWWW!')
|
||||
print('PASS but with duplicate muts: OR dfs successfully combined! PHEWWWW!'
|
||||
, '\nDuplicate muts present but with different \'ref\' and \'alt\' alleles')
|
||||
else:
|
||||
print('FAIL: could not combine OR dfs'
|
||||
, '\nCheck expected rows and cols calculation and join type')
|
||||
|
||||
print('Dim of merged ors_df:', ors_df.shape)
|
||||
|
||||
ors_df[merging_cols_m5].apply(len)
|
||||
ors_df[merging_cols_m5].apply(len) == len(ors_df)
|
||||
|
||||
#%%============================================================================
|
||||
# formatting ors_df
|
||||
|
||||
ors_df.columns
|
||||
|
||||
|
||||
# Dropping unncessary columns: already removed in ealier preprocessing
|
||||
#cols_to_drop = ['reference_allele', 'alternate_allele', 'symbol' ]
|
||||
cols_to_drop = ['n_miss']
|
||||
print('Dropping', len(cols_to_drop), 'columns:\n'
|
||||
, cols_to_drop)
|
||||
|
|
@ -283,18 +311,13 @@ column_order = ['mutation'
|
|||
, 'wild_type'
|
||||
, 'position'
|
||||
, 'mutant_type'
|
||||
#, 'chr_num_allele' #old
|
||||
, 'ref_allele'
|
||||
, 'alt_allele'
|
||||
, 'mut_info_f1'
|
||||
, 'mut_info_f2'
|
||||
, 'mut_type'
|
||||
, 'gene_id'
|
||||
#, 'gene_number' #old
|
||||
, 'gene_name'
|
||||
#, 'mut_region'
|
||||
#, 'reference_allele'
|
||||
#, 'alternate_allele'
|
||||
, 'chromosome_number'
|
||||
, 'af'
|
||||
, 'af_kin'
|
||||
|
|
@ -317,14 +340,9 @@ column_order = ['mutation'
|
|||
, 'se_kin'
|
||||
, 'zval_logistic'
|
||||
, 'logl_h1_kin'
|
||||
, 'l_remle_kin'
|
||||
#, 'wt_3let' # old
|
||||
#, 'mt_3let' # old
|
||||
#, 'symbol'
|
||||
#, 'n_miss'
|
||||
]
|
||||
, 'l_remle_kin']
|
||||
|
||||
if ( (len(column_order) == ors_df.shape[1]) and (DataFrame(column_order).isin(ors_df.columns).all().all()):
|
||||
if ( (len(column_order) == ors_df.shape[1]) and (DataFrame(column_order).isin(ors_df.columns).all().all())):
|
||||
print('PASS: Column order generated for all:', len(column_order), 'columns'
|
||||
, '\nColumn names match, safe to reorder columns'
|
||||
, '\nApplying column order to df...' )
|
||||
|
|
@ -337,6 +355,61 @@ else:
|
|||
|
||||
print('\nResult of Sixth merge:', ors_df_ordered.shape
|
||||
, '\n===================================================================')
|
||||
#%%
|
||||
ors_df_ordered.shape
|
||||
check = ors_df_ordered[['mutationinformation','mutation', 'wild_type', 'position', 'mutant_type']]
|
||||
|
||||
# populating 'nan' info
|
||||
lookup_dict = dict()
|
||||
for k, v in low_3letter_dict.items():
|
||||
lookup_dict[k] = v['one_letter_code']
|
||||
#print(lookup_dict)
|
||||
|
||||
wt = ors_df_ordered['mutation'].str.extract(wt_regex).squeeze()
|
||||
#print(wt)
|
||||
ors_df_ordered['wild_type'] = wt.map(lookup_dict)
|
||||
|
||||
ors_df_ordered['position'] = ors_df_ordered['mutation'].str.extract(pos_regex)
|
||||
|
||||
mt = ors_df_ordered['mutation'].str.extract(mut_regex).squeeze()
|
||||
ors_df_ordered['mutant_type'] = mt.map(lookup_dict)
|
||||
|
||||
ors_df_ordered['mutationinformation'] = ors_df_ordered['wild_type'] + ors_df_ordered.position.map(str) + ors_df_ordered['mutant_type']
|
||||
check = ors_df_ordered[['mutationinformation','mutation', 'wild_type', 'position', 'mutant_type']]
|
||||
|
||||
# populate mut_info_f1
|
||||
ors_df_ordered['mut_info_f1'].isna().sum()
|
||||
ors_df_ordered['mut_info_f1'] = ors_df_ordered['position'].astype(str) + ors_df_ordered['wild_type'] + '>' + ors_df_ordered['position'].astype(str) + ors_df_ordered['mutant_type']
|
||||
ors_df_ordered['mut_info_f1'].isna().sum()
|
||||
|
||||
# populate mut_info_f2
|
||||
ors_df_ordered['mut_info_f2'] = ors_df_ordered['mutation'].str.replace(gene_match.lower(), 'p.', regex = True)
|
||||
|
||||
# populate mut_type
|
||||
ors_df_ordered['mut_type'].isna().sum()
|
||||
#mut_type_word = ors_df_ordered['mut_type'].value_counts()
|
||||
mut_type_word = 'missense' # FIXME, should be derived
|
||||
ors_df_ordered['mut_type'].fillna(mut_type_word, inplace = True)
|
||||
ors_df_ordered['mut_type'].isna().sum()
|
||||
|
||||
# populate gene_id
|
||||
ors_df_ordered['gene_id'].isna().sum()
|
||||
#gene_id_word = ors_df_ordered['gene_id'].value_counts()
|
||||
gene_id_word = 'Rv2043c' # FIXME, should be derived
|
||||
ors_df_ordered['gene_id'].fillna(gene_id_word, inplace = True)
|
||||
ors_df_ordered['gene_id'].isna().sum()
|
||||
|
||||
# populate gene_name
|
||||
ors_df_ordered['gene_name'].isna().sum()
|
||||
ors_df_ordered['gene_name'].value_counts()
|
||||
ors_df_ordered['gene_name'].fillna(gene, inplace = True)
|
||||
ors_df_ordered['gene_name'].isna().sum()
|
||||
|
||||
# check numbers
|
||||
ors_df_ordered['or_kin'].isna().sum()
|
||||
# should be 0
|
||||
ors_df_ordered['or_mychisq'].isna().sum()
|
||||
|
||||
#%%============================================================================
|
||||
print('==================================='
|
||||
, '\nSixth merge: Fourth + Fifth merge'
|
||||
|
|
@ -344,53 +417,159 @@ print('==================================='
|
|||
, '\n===================================')
|
||||
|
||||
#combined_df_all = combine_dfs_with_checks(combined_df, ors_df_ordered, my_join = i_join)
|
||||
merging_cols_m6 = detect_common_cols(combined_df, ors_df_ordered)
|
||||
merging_cols_m6 = detect_common_cols(combined_df, ors_df_ordered)
|
||||
|
||||
# dtype problems
|
||||
if len(merging_cols_m6) > 1 and 'position'in merging_cols_m6:
|
||||
print('Removing \'position\' from merging_cols_m6 to make dtypes consistent'
|
||||
, '\norig length of merging_cols_m6:', len(merging_cols_m6))
|
||||
merging_cols_m6.remove('position')
|
||||
print('\nlength after removing:', len(merging_cols_m6))
|
||||
|
||||
print('Dim of df1:', combined_df.shape
|
||||
, '\nDim of df2:', ors_df_ordered.shape
|
||||
, '\nNo. of merging_cols:', len(merging_cols_m6))
|
||||
|
||||
print('Checking mutations in the two dfs:'
|
||||
, '\nmuts in df1 but NOT in df2:'
|
||||
, '\nmuts in df1 present in df2:'
|
||||
, combined_df['mutationinformation'].isin(ors_df_ordered['mutationinformation']).sum()
|
||||
, '\nmuts in df2 but NOT in df1:'
|
||||
, '\nmuts in df2 present in df1:'
|
||||
, ors_df_ordered['mutationinformation'].isin(combined_df['mutationinformation']).sum())
|
||||
|
||||
#print('\nNo. of common muts:', np.intersect1d(combined_df['mutationinformation'], ors_df_ordered['mutationinformation']) )
|
||||
|
||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
combined_df_all = pd.merge(combined_df, ors_df, on = merging_cols_m6, how = l_join)
|
||||
#----------
|
||||
# merge 6
|
||||
#----------
|
||||
combined_df_all = pd.merge(combined_df, ors_df_ordered, on = merging_cols_m6, how = o_join)
|
||||
combined_df_all.shape
|
||||
|
||||
# FIXME: DIM
|
||||
# only with left join!
|
||||
outdf_expected_rows = len(combined_df)
|
||||
# sanity check for merge 6
|
||||
outdf_expected_rows = len(combined_df) + extra_muts_myor
|
||||
unique_muts = len(combined_df)
|
||||
outdf_expected_cols = len(combined_df.columns) + len(ors_df_ordered.columns) - len(merging_cols_m6)
|
||||
|
||||
#if combined_df_all.shape[1] == outdf_expected_cols and combined_df_all.shape[0] == outdf_expected_rows:
|
||||
if combined_df_all.shape[1] == outdf_expected_cols and combined_df_all['mutationinformation'].nunique() == outdf_expected_rows:
|
||||
if combined_df_all.shape[0] == outdf_expected_rows and combined_df_all.shape[1] == outdf_expected_cols and combined_df_all['mutationinformation'].nunique() == unique_muts:
|
||||
print('PASS: Df dimension match'
|
||||
, '\nDim of combined_df_all with join type:', l_join
|
||||
, '\ncombined_df_all with join type:', o_join
|
||||
, '\n', combined_df_all.shape
|
||||
, '\n===============================================================')
|
||||
else:
|
||||
print('FAIL: Df dimension mismatch'
|
||||
, 'Cannot generate expected dim. See details of merge performed'
|
||||
, '\ndf1 dim:', combined_df.shape
|
||||
, '\ndf2 dim:', ors_df.shape
|
||||
, '\ndf2 dim:', ors_df_ordered.shape
|
||||
, '\nGot:', combined_df_all.shape
|
||||
, '\nmuts in df1 but NOT in df2:'
|
||||
, combined_df['mutationinformation'].isin(ors_df['mutationinformation']).sum()
|
||||
, combined_df['mutationinformation'].isin(ors_df_ordered['mutationinformation']).sum()
|
||||
, '\nmuts in df2 but NOT in df1:'
|
||||
, ors_df['mutationinformation'].isin(combined_df['mutationinformation']).sum())
|
||||
sys.exit()
|
||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
# nan in mutation col
|
||||
# FIXME: should get fixmed with JP's resolved dataset!?
|
||||
combined_df_all['mutation'].isna().sum()
|
||||
baz = combined_df_all[combined_df_all['mutation'].isna()]
|
||||
|
||||
# drop extra cols
|
||||
all_cols = combined_df_all.columns
|
||||
|
||||
#pos_cols_check = combined_df_all[['position_x','position_y']]
|
||||
c = combined_df_all[['position_x','position_y']].isna().sum()
|
||||
pos_col_to_drop = c.index[c>0].to_list()
|
||||
cols_to_drop = pos_col_to_drop + ['wild_type_kd']
|
||||
|
||||
print('Dropping', len(cols_to_drop), 'columns:\n', cols_to_drop)
|
||||
combined_df_all.drop(cols_to_drop, axis = 1, inplace = True)
|
||||
|
||||
# rename position_x to position
|
||||
pos_col_to_rename = c.index[c==0].to_list()
|
||||
combined_df_all.shape
|
||||
combined_df_all.rename(columns = { pos_col_to_rename[0]: 'position'}, inplace = True)
|
||||
combined_df_all.shape
|
||||
|
||||
all_cols = combined_df_all.columns
|
||||
|
||||
#%% reorder cols to for convenience
|
||||
first_cols = ['mutationinformation','mutation', 'wild_type', 'position', 'mutant_type']
|
||||
last_cols = [col for col in combined_df_all.columns if col not in first_cols]
|
||||
|
||||
combined_df_all = combined_df_all[first_cols+last_cols]
|
||||
|
||||
#%% IMPORTANT: check if mutation related info is all populated after this merge
|
||||
# select string colnames to ensure no NA exist there
|
||||
string_cols = combined_df_all.columns[combined_df_all.applymap(lambda x: isinstance(x, str)).all(0)]
|
||||
|
||||
if (combined_df_all[string_cols].isna().sum(axis = 0)== 0).all():
|
||||
print('PASS: All string cols are populated with no NAs')
|
||||
else:
|
||||
print('FAIL: NAs detected in string cols')
|
||||
print(combined_df_all[string_cols].isna().sum(axis = 0))
|
||||
sys.exit()
|
||||
|
||||
# relevant mut cols
|
||||
check_mut_cols = merging_cols_m5 + merging_cols_m6
|
||||
|
||||
count_na_mut_cols = combined_df_all[check_mut_cols].isna().sum().reset_index().rename(columns = {'index': 'col_name', 0: 'na_count'})
|
||||
print(check_mut_cols)
|
||||
|
||||
c2 = combined_df_all[check_mut_cols].isna().sum()
|
||||
missing_info_cols = c2.index[c2>0].to_list()
|
||||
|
||||
if c2.sum()>0:
|
||||
#na_muts_n = combined_df_all['mutation'].isna().sum()
|
||||
na_muts_n = combined_df_all[missing_info_cols].isna().sum()
|
||||
print(na_muts_n.values[0], 'mutations have missing \'mutation\' info.'
|
||||
, '\nFetching these from reference dict...')
|
||||
else:
|
||||
print('No missing \'mutation\' has been detected!')
|
||||
|
||||
lookup_dict = dict()
|
||||
for k, v in oneletter_aa_dict.items():
|
||||
lookup_dict[k] = v['three_letter_code_lower']
|
||||
print(lookup_dict)
|
||||
wt_3let = combined_df_all['wild_type'].map(lookup_dict)
|
||||
#print(wt_3let)
|
||||
pos = combined_df_all['position'].astype(str)
|
||||
#print(pos)
|
||||
mt_3let = combined_df_all['mutant_type'].map(lookup_dict)
|
||||
#print(mt_3let)
|
||||
# override the 'mutation' column
|
||||
combined_df_all['mutation'] = 'pnca_p.' + wt_3let + pos + mt_3let
|
||||
print(combined_df_all['mutation'])
|
||||
|
||||
# check again
|
||||
if combined_df_all[missing_info_cols].isna().sum().all() == 0:
|
||||
print('PASS: No mutations have missing \'mutation\' info.')
|
||||
else:
|
||||
print('FAIL:', combined_df_all[missing_info_cols].isna().sum().values[0]
|
||||
, '\nmutations have missing info STILL...')
|
||||
sys.exit()
|
||||
|
||||
#%% check
|
||||
foo = combined_df_all.drop_duplicates('mutationinformation')
|
||||
foo2 = combined_df_all.drop_duplicates('mutation')
|
||||
if foo.equals(foo2):
|
||||
print('PASS: Dropping mutation or mutatationinformation has the same effect\n')
|
||||
else:
|
||||
print('FAIL: Still problems in merged data')
|
||||
sys.exit()
|
||||
|
||||
#%%============================================================================
|
||||
output_cols = combined_df_all.columns
|
||||
print('Output cols:', output_cols)
|
||||
|
||||
#%% IMPORTANT result info
|
||||
if combined_df_all['or_mychisq'].isna().sum() == len(combined_df) - len(afor_df):
|
||||
print('PASS: No. of NA in or_mychisq matches expected length'
|
||||
, '\nNo. of with NA in or_mychisq:', combined_df_all['or_mychisq'].isna().sum()
|
||||
, '\nNo. of NA in or_kin:', combined_df_all['or_kin'].isna().sum())
|
||||
else:
|
||||
print('FAIL: No. of NA in or_mychisq does not match expected length')
|
||||
|
||||
|
||||
if combined_df_all.shape[0] == outdf_expected_rows:
|
||||
print('\nINFORMARIONAL ONLY: combined_df_all has duplicate muts present but with unique ref and alt allele'
|
||||
, '\n=============================================================')
|
||||
else:
|
||||
print('combined_df_all has no duplicate muts present'
|
||||
,'\n===============================================================')
|
||||
|
||||
print('\nDim of combined_data:', combined_df_all.shape
|
||||
, '\nNo. of unique mutations:', combined_df_all['mutationinformation'].nunique())
|
||||
|
||||
|
||||
#%%============================================================================
|
||||
# write csv
|
||||
|
|
|
|||
|
|
@ -45,8 +45,6 @@ Created on Tue Aug 6 12:56:03 2019
|
|||
#5. chain
|
||||
#6. wild_pos
|
||||
#7. wild_chain_pos
|
||||
|
||||
|
||||
#=======================================================================
|
||||
#%% load libraries
|
||||
import os, sys
|
||||
|
|
@ -83,16 +81,16 @@ gene = args.gene
|
|||
gene_match = gene + '_p.'
|
||||
print('mut pattern for gene', gene, ':', gene_match)
|
||||
|
||||
nssnp_match = gene_match +'[A-Z]{3}[0-9]+[A-Z]{3}'
|
||||
nssnp_match = gene_match +'[A-Za-z]{3}[0-9]+[A-Za-z]{3}'
|
||||
print('nsSNP for gene', gene, ':', nssnp_match)
|
||||
|
||||
wt_regex = gene_match.lower()+'(\w{3})'
|
||||
wt_regex = gene_match.lower()+'([A-Za-z]{3})'
|
||||
print('wt regex:', wt_regex)
|
||||
|
||||
mut_regex = r'\d+(\w{3})$'
|
||||
mut_regex = r'[0-9]+(\w{3})$'
|
||||
print('mt regex:', mut_regex)
|
||||
|
||||
pos_regex = r'(\d+)'
|
||||
pos_regex = r'([0-9]+)'
|
||||
print('position regex:', pos_regex)
|
||||
|
||||
# building cols to extract
|
||||
|
|
@ -156,30 +154,29 @@ if in_filename_master == 'original_tanushree_data_v2.csv':
|
|||
else:
|
||||
core_cols = ['id'
|
||||
, 'sample'
|
||||
, 'patient_id'
|
||||
, 'strain'
|
||||
#, 'patient_id'
|
||||
#, 'strain'
|
||||
, 'lineage'
|
||||
, 'sublineage'
|
||||
, 'country'
|
||||
#, 'country'
|
||||
, 'country_code'
|
||||
, 'geographic_source'
|
||||
#, 'region'
|
||||
, 'location'
|
||||
, 'host_body_site'
|
||||
, 'environment_material'
|
||||
, 'host_status'
|
||||
, 'host_sex'
|
||||
, 'submitted_host_sex'
|
||||
, 'hiv_status'
|
||||
, 'HIV_status'
|
||||
, 'tissue_type'
|
||||
, 'isolation_source'
|
||||
#, 'location'
|
||||
#, 'host_body_site'
|
||||
#, 'environment_material'
|
||||
#, 'host_status'
|
||||
#, 'host_sex'
|
||||
#, 'submitted_host_sex'
|
||||
#, 'hiv_status'
|
||||
#, 'HIV_status'
|
||||
#, 'tissue_type'
|
||||
#, 'isolation_source'
|
||||
, resistance_col]
|
||||
|
||||
variable_based_cols = [drug
|
||||
, dr_muts_col
|
||||
, other_muts_col]
|
||||
#, resistance_col]
|
||||
|
||||
cols_to_extract = core_cols + variable_based_cols
|
||||
print('Extracting', len(cols_to_extract), 'columns from master data')
|
||||
|
|
@ -202,7 +199,7 @@ print('No. of NAs/column:' + '\n', meta_data.isna().sum()
|
|||
|
||||
#%% Write check file
|
||||
check_file = outdir + '/' + gene.lower() + '_gwas.csv'
|
||||
meta_data.to_csv(check_file)
|
||||
meta_data.to_csv(check_file, index = False)
|
||||
print('Writing subsetted gwas data'
|
||||
, '\nFile', check_file
|
||||
, '\nDim:', meta_data.shape)
|
||||
|
|
@ -217,9 +214,9 @@ print('Writing subsetted gwas data'
|
|||
# drug counts: complete samples for OR calcs
|
||||
meta_data[drug].value_counts()
|
||||
print('===========================================================\n'
|
||||
, 'RESULT: No. of Sus and Res samples:\n', meta_data[drug].value_counts()
|
||||
, 'RESULT: No. of Sus and Res', drug, 'samples:\n', meta_data[drug].value_counts()
|
||||
, '\n===========================================================\n'
|
||||
, 'RESULT: Percentage of Sus and Res samples:\n', meta_data[drug].value_counts(normalize = True)*100
|
||||
, 'RESULT: Percentage of Sus and Res', drug, 'samples:\n', meta_data[drug].value_counts(normalize = True)*100
|
||||
, '\n===========================================================')
|
||||
|
||||
#%%
|
||||
|
|
@ -1173,7 +1170,7 @@ del(out_filename_metadata_poscounts)
|
|||
|
||||
#%% Write file: gene_metadata (i.e gene_LF1)
|
||||
# where each row has UNIQUE mutations NOT unique sample ids
|
||||
out_filename_metadata = gene.lower() + '_metadata.csv'
|
||||
out_filename_metadata = gene.lower() + '_metadata_raw.csv'
|
||||
outfile_metadata = outdir + '/' + out_filename_metadata
|
||||
print('Writing file: LF formatted data'
|
||||
, '\nFile:', outfile_metadata
|
||||
|
|
|
|||
211
scripts/ks_test_PS.R
Normal file
211
scripts/ks_test_PS.R
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
#!/usr/bin/env Rscript
|
||||
#########################################################
|
||||
# TASK: KS test for PS/DUET lineage distributions
|
||||
#=======================================================================
|
||||
#=======================================================================
|
||||
# working dir and loading libraries
|
||||
getwd()
|
||||
setwd("~/git/LSHTM_analysis/scripts/")
|
||||
getwd()
|
||||
|
||||
#source("/plotting/Header_TT.R")
|
||||
#source("../barplot_colour_function.R")
|
||||
#require(data.table)
|
||||
source("plotting/combining_dfs_plotting.R")
|
||||
# should return the following dfs, directories and variables
|
||||
|
||||
# PS combined:
|
||||
# 1) merged_df2
|
||||
# 2) merged_df2_comp
|
||||
# 3) merged_df3
|
||||
# 4) merged_df3_comp
|
||||
|
||||
# LIG combined:
|
||||
# 5) merged_df2_lig
|
||||
# 6) merged_df2_comp_lig
|
||||
# 7) merged_df3_lig
|
||||
# 8) merged_df3_comp_lig
|
||||
|
||||
# 9) my_df_u
|
||||
# 10) my_df_u_lig
|
||||
|
||||
cat(paste0("Directories imported:"
|
||||
, "\ndatadir:", datadir
|
||||
, "\nindir:", indir
|
||||
, "\noutdir:", outdir
|
||||
, "\nplotdir:", plotdir))
|
||||
|
||||
cat(paste0("Variables imported:"
|
||||
, "\ndrug:", drug
|
||||
, "\ngene:", gene
|
||||
, "\ngene_match:", gene_match
|
||||
, "\nAngstrom symbol:", angstroms_symbol
|
||||
, "\nNo. of duplicated muts:", dup_muts_nu
|
||||
, "\nNA count for ORs:", na_count
|
||||
, "\nNA count in df2:", na_count_df2
|
||||
, "\nNA count in df3:", na_count_df3))
|
||||
|
||||
###########################
|
||||
# Data for stats
|
||||
# you need merged_df2 or merged_df2_comp
|
||||
# since this is one-many relationship
|
||||
# i.e the same SNP can belong to multiple lineages
|
||||
# using the _comp dataset means
|
||||
# we lose some muts and at this level, we should use
|
||||
# as much info as available, hence use df with NA
|
||||
###########################
|
||||
|
||||
# REASSIGNMENT
|
||||
my_df = merged_df2
|
||||
|
||||
# delete variables not required
|
||||
rm(merged_df2, merged_df2_comp, merged_df3, merged_df3_comp)
|
||||
|
||||
# quick checks
|
||||
colnames(my_df)
|
||||
str(my_df)
|
||||
|
||||
# Ensure correct data type in columns to plot: need to be factor
|
||||
is.factor(my_df$lineage)
|
||||
my_df$lineage = as.factor(my_df$lineage)
|
||||
is.factor(my_df$lineage)
|
||||
|
||||
table(my_df$mutation_info); str(my_df$mutation_info)
|
||||
|
||||
# subset df with dr muts only
|
||||
my_df_dr = subset(my_df, mutation_info == "dr_mutations_pyrazinamide")
|
||||
table(my_df_dr$mutation_info)
|
||||
|
||||
# stats for all muts and dr_muts
|
||||
# 1) for all muts
|
||||
# 2) for dr_muts
|
||||
#===========================
|
||||
table(my_df$lineage); str(my_df$lineage)
|
||||
table(my_df_dr$lineage); str(my_df_dr$lineage)
|
||||
|
||||
# subset only lineages1-4
|
||||
sel_lineages = c("lineage1"
|
||||
, "lineage2"
|
||||
, "lineage3"
|
||||
, "lineage4")
|
||||
|
||||
# subset and refactor: all muts
|
||||
df_lin = subset(my_df, subset = lineage %in% sel_lineages)
|
||||
df_lin$lineage = factor(df_lin$lineage)
|
||||
|
||||
# subset and refactor: dr muts
|
||||
df_lin_dr = subset(my_df_dr, subset = lineage %in% sel_lineages)
|
||||
df_lin_dr$lineage = factor(df_lin_dr$lineage)
|
||||
|
||||
|
||||
#{RESULT: No of samples within lineage}
|
||||
table(df_lin$lineage)
|
||||
table(df_lin_dr$lineage)
|
||||
|
||||
#{Result: No. of unique mutations the 4 lineages contribute to}
|
||||
length(unique(df_lin$mutationinformation))
|
||||
length(unique(df_lin_dr$mutationinformation))
|
||||
|
||||
|
||||
# COMPARING DISTRIBUTIONS
|
||||
#================
|
||||
# ALL mutations
|
||||
#=================
|
||||
head(df_lin$lineage)
|
||||
df_lin$lineage = as.character(df_lin$lineage)
|
||||
|
||||
lin1 = df_lin[df_lin$lineage == "lineage1",]$duet_scaled
|
||||
lin2 = df_lin[df_lin$lineage == "lineage2",]$duet_scaled
|
||||
lin3 = df_lin[df_lin$lineage == "lineage3",]$duet_scaled
|
||||
lin4 = df_lin[df_lin$lineage == "lineage4",]$duet_scaled
|
||||
|
||||
# ks test
|
||||
lin12 = ks.test(lin1,lin2)
|
||||
lin12_df = as.data.frame(cbind(lin12$data.name, lin12$p.value))
|
||||
|
||||
lin13 = ks.test(lin1,lin3)
|
||||
lin13_df = as.data.frame(cbind(lin13$data.name, lin13$p.value))
|
||||
|
||||
lin14 = ks.test(lin1,lin4)
|
||||
lin14_df = as.data.frame(cbind(lin14$data.name, lin14$p.value))
|
||||
|
||||
lin23 = ks.test(lin2,lin3)
|
||||
lin23_df = as.data.frame(cbind(lin23$data.name, lin23$p.value))
|
||||
|
||||
lin24 = ks.test(lin2,lin4)
|
||||
lin24_df = as.data.frame(cbind(lin24$data.name, lin24$p.value))
|
||||
|
||||
lin34 = ks.test(lin3,lin4)
|
||||
lin34_df = as.data.frame(cbind(lin34$data.name, lin34$p.value))
|
||||
|
||||
ks_results_all = rbind(lin12_df
|
||||
, lin13_df
|
||||
, lin14_df
|
||||
, lin23_df
|
||||
, lin24_df
|
||||
, lin34_df)
|
||||
|
||||
#p-value < 2.2e-16
|
||||
rm(lin12, lin12_df
|
||||
, lin13, lin13_df
|
||||
, lin14, lin14_df
|
||||
, lin23, lin23_df
|
||||
, lin24, lin24_df
|
||||
, lin34, lin34_df)
|
||||
|
||||
#================
|
||||
# DRUG mutations
|
||||
#=================
|
||||
head(df_lin_dr$lineage)
|
||||
df_lin_dr$lineage = as.character(df_lin_dr$lineage)
|
||||
|
||||
lin1_dr = df_lin_dr[df_lin_dr$lineage == "lineage1",]$duet_scaled
|
||||
lin2_dr = df_lin_dr[df_lin_dr$lineage == "lineage2",]$duet_scaled
|
||||
lin3_dr = df_lin_dr[df_lin_dr$lineage == "lineage3",]$duet_scaled
|
||||
lin4_dr = df_lin_dr[df_lin_dr$lineage == "lineage4",]$duet_scaled
|
||||
|
||||
# ks test: dr muts
|
||||
lin12_dr = ks.test(lin1_dr,lin2_dr)
|
||||
lin12_df_dr = as.data.frame(cbind(lin12_dr$data.name, lin12_dr$p.value))
|
||||
|
||||
lin13_dr = ks.test(lin1_dr,lin3_dr)
|
||||
lin13_df_dr = as.data.frame(cbind(lin13_dr$data.name, lin13_dr$p.value))
|
||||
|
||||
lin14_dr = ks.test(lin1_dr,lin4_dr)
|
||||
lin14_df_dr = as.data.frame(cbind(lin14_dr$data.name, lin14_dr$p.value))
|
||||
|
||||
lin23_dr = ks.test(lin2_dr,lin3_dr)
|
||||
lin23_df_dr = as.data.frame(cbind(lin23_dr$data.name, lin23_dr$p.value))
|
||||
|
||||
lin24_dr = ks.test(lin2_dr,lin4_dr)
|
||||
lin24_df_dr = as.data.frame(cbind(lin24_dr$data.name, lin24_dr$p.value))
|
||||
|
||||
lin34_dr = ks.test(lin3_dr,lin4_dr)
|
||||
lin34_df_dr = as.data.frame(cbind(lin34_dr$data.name, lin34_dr$p.value))
|
||||
|
||||
ks_results_dr = rbind(lin12_df_dr
|
||||
, lin13_df_dr
|
||||
, lin14_df_dr
|
||||
, lin23_df_dr
|
||||
, lin24_df_dr
|
||||
, lin34_df_dr)
|
||||
|
||||
ks_results_combined = cbind(ks_results_all, ks_results_dr)
|
||||
|
||||
my_colnames = c("Lineage_comparisons"
|
||||
, paste0("All_mutations n=", nrow(df_lin))
|
||||
, paste0("Drug_associated_mutations n=", nrow(df_lin_dr)))
|
||||
my_colnames
|
||||
|
||||
# select the output columns
|
||||
ks_results_combined_f = ks_results_combined[,c(1,2,4)]
|
||||
|
||||
colnames(ks_results_combined_f) = my_colnames
|
||||
ks_results_combined_f
|
||||
|
||||
#=============
|
||||
# write output file
|
||||
#=============
|
||||
ks_results = paste0(outdir,"/results/ks_results.csv")
|
||||
write.csv(ks_results_combined_f, ks_results, row.names = F)
|
||||
|
||||
|
|
@ -104,7 +104,7 @@ or_df.columns
|
|||
|
||||
#%% snp_info file: master and gene specific ones
|
||||
# gene info
|
||||
info_df2 = pd.read_csv(gene_info, sep = '\t', header = 0) #447, 10
|
||||
info_df2 = pd.read_csv(gene_info, sep = '\t', header = 0) #447, 11
|
||||
#info_df2 = pd.read_csv(gene_info, sep = ',', header = 0) #447 10
|
||||
mis_mut_cover = (info_df2['chromosome_number'].nunique()/info_df2['chromosome_number'].count()) * 100
|
||||
print('*****RESULT*****'
|
||||
|
|
@ -212,7 +212,7 @@ else:
|
|||
|
||||
#PENDING Jody's reply
|
||||
# !!!!!!!!
|
||||
# drop nan from dfm2_mis as these are not useful
|
||||
# drop nan from dfm2_mis as these are not useful and JP confirmed the same
|
||||
print('Dropping NAs before further processing...')
|
||||
dfm2_mis = dfm2[dfm2['mut_type'].notnull()]
|
||||
# !!!!!!!!
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
#=======================================================================
|
||||
#%% useful links
|
||||
#https://towardsdatascience.com/autoviz-automatically-visualize-any-dataset-ba2691a8b55a
|
||||
#https://pypi.org/project/autoviz/
|
||||
#=======================================================================
|
||||
import os, sys
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
import argparse
|
||||
from autoviz.AutoViz_Class import AutoViz_Class
|
||||
|
||||
homedir = os.path.expanduser('~')
|
||||
os.chdir(homedir + '/git/LSHTM_analysis/scripts')
|
||||
#%%============================================================================
|
||||
# variables
|
||||
gene = 'pncA'
|
||||
drug = 'pyrazinamide'
|
||||
|
||||
#%%============================================================================
|
||||
#==============
|
||||
# directories
|
||||
#==============
|
||||
datadir = homedir + '/' + 'git/Data'
|
||||
|
||||
indir = datadir + '/' + drug + '/input'
|
||||
|
||||
outdir = datadir + '/' + drug + '/output'
|
||||
|
||||
#=======
|
||||
# input
|
||||
#=======
|
||||
in_filename_plotting = 'car_design.csv'
|
||||
in_filename_plotting = gene.lower() + '_all_params.csv'
|
||||
infile_plotting = outdir + '/' + in_filename_plotting
|
||||
print('plotting file: ', infile_plotting
|
||||
, '\n============================================================')
|
||||
#=======================================================================
|
||||
plotting_df = pd.read_csv(infile_plotting, sep = ',')
|
||||
#Instantiate the AutoViz class
|
||||
AV = AutoViz_Class()
|
||||
df = AV.AutoViz(infile_plotting)
|
||||
#df2 = AV.AutoViz(plotting_df)
|
||||
plotting_df.columns[~plotting_df.columns.isin(df.columns)]
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
#!/usr/bin/env Rscript
|
||||
getwd()
|
||||
#!/usr/bin/env Rscript getwd()
|
||||
setwd("~/git/LSHTM_analysis/scripts/plotting")
|
||||
getwd()
|
||||
|
||||
|
|
@ -19,8 +18,7 @@ getwd()
|
|||
library(ggplot2)
|
||||
library(data.table)
|
||||
source("barplot_colour_function.R")
|
||||
#source("subcols_axis.R")
|
||||
source("subcols_axis_PS.R")
|
||||
source("subcols_axis.R")
|
||||
|
||||
# should return the following dfs, directories and variables
|
||||
# mut_pos_cols
|
||||
|
|
@ -161,9 +159,7 @@ min(df$duet_scaled)
|
|||
max(df$duet_scaled)
|
||||
|
||||
# sanity checks
|
||||
# very important!!!!
|
||||
tapply(df$duet_scaled, df$duet_outcome, min)
|
||||
|
||||
tapply(df$duet_scaled, df$duet_outcome, max)
|
||||
|
||||
# My colour FUNCTION: based on group and subgroup
|
||||
|
|
@ -241,7 +237,7 @@ outPlot = g +
|
|||
, axis.ticks.x = element_blank()) +
|
||||
labs(title = ""
|
||||
#title = my_title
|
||||
, x = "position"
|
||||
, x = "Position"
|
||||
, y = "Frequency")
|
||||
|
||||
print(outPlot)
|
||||
|
|
|
|||
|
|
@ -1,87 +0,0 @@
|
|||
,x,,changes,
|
||||
1,mutationinformation,,Mutationinformation,
|
||||
2,wild_type,,,consider...wild_aa
|
||||
3,position,,Position,
|
||||
4,mutant_type,,,consider...mutant_aa
|
||||
5,chain,,,
|
||||
6,ligand_id,,,
|
||||
7,ligand_distance,,,
|
||||
8,duet_stability_change,,,
|
||||
9,duet_outcome,,DUET_outcome,
|
||||
10,ligand_affinity_change,,,
|
||||
11,ligand_outcome,,Lig_outcome,
|
||||
12,duet_scaled,,ratioDUET,
|
||||
13,affinity_scaled,,ratioPredAff,
|
||||
14,wild_pos,,WildPos,
|
||||
15,wild_chain_pos,,,
|
||||
16,ddg,,,
|
||||
17,contacts,,,
|
||||
18,electro_rr,,,
|
||||
19,electro_mm,,,
|
||||
20,electro_sm,,,
|
||||
21,electro_ss,,,
|
||||
22,disulfide_rr,,,
|
||||
23,disulfide_mm,,,
|
||||
24,disulfide_sm,,,
|
||||
25,disulfide_ss,,,
|
||||
26,hbonds_rr,,,
|
||||
27,hbonds_mm,,,
|
||||
28,hbonds_sm,,,
|
||||
29,hbonds_ss,,,
|
||||
30,partcov_rr,,,
|
||||
31,partcov_mm,,,
|
||||
32,partcov_sm,,,
|
||||
33,partcov_ss,,,
|
||||
34,vdwclashes_rr,,,
|
||||
35,vdwclashes_mm,,,
|
||||
36,vdwclashes_sm,,,
|
||||
37,vdwclashes_ss,,,
|
||||
38,volumetric_rr,,,
|
||||
39,volumetric_mm,,,
|
||||
40,volumetric_sm,,,
|
||||
41,volumetric_ss,,,
|
||||
42,wild_type_dssp,,,
|
||||
43,asa,,,
|
||||
44,rsa,,,
|
||||
45,ss,,,
|
||||
46,ss_class,,,
|
||||
47,chain_id,,,
|
||||
48,wild_type_kd,,,
|
||||
49,kd_values,,,
|
||||
50,rd_values,,,
|
||||
51,wt_3letter_caps,,,
|
||||
52,mutation,,,
|
||||
53,af,,,
|
||||
54,beta_logistic,,,
|
||||
55,or_logistic,,,
|
||||
56,pval_logistic,,,
|
||||
57,se_logistic,,,
|
||||
58,zval_logistic,,,
|
||||
59,ci_low_logistic,,,
|
||||
60,ci_hi_logistic,,,
|
||||
61,or_mychisq,,,
|
||||
62,or_fisher,,,
|
||||
63,pval_fisher,,,
|
||||
64,ci_low_fisher,,,
|
||||
65,ci_hi_fisher,,,
|
||||
66,est_chisq,,,
|
||||
67,pval_chisq,,,
|
||||
68,chromosome_number,,,
|
||||
69,ref_allele,,,
|
||||
70,alt_allele,,,
|
||||
71,mut_type,,,
|
||||
72,gene_id,,,
|
||||
73,gene_number,,,
|
||||
74,mut_region,,,
|
||||
75,mut_info,,,
|
||||
76,chr_num_allele,,,
|
||||
77,wt_3let,,,
|
||||
78,mt_3let,,,
|
||||
79,af_kin,,,
|
||||
80,or_kin,,,
|
||||
81,pwald_kin,,,
|
||||
82,beta_kin,,,
|
||||
83,se_kin,,,
|
||||
84,logl_h1_kin,,,
|
||||
85,l_remle_kin,,,
|
||||
86,n_miss,,,
|
||||
500Internal server error An error occurred: Render failed, failed to render template: repo/pulls/files, error: template error: builtin(bindata):repo/diff/csv_diff:3:15 : executing "repo/diff/csv_diff" at <call .root.CreateCsvDiff .file .blobBase .blobHead>: error calling call: runtime error: invalid memory address or nil pointer dereference
----------------------------------------------------------------------
{{$result := call .root.CreateCsvDiff .file .blobBase .blobHead}}
^
----------------------------------------------------------------------
Forgejo version: 11.0.1+gitea-1.22.0 |