combined and output all ors
This commit is contained in:
parent
003b22ce3f
commit
a298071309
2 changed files with 185 additions and 144 deletions
|
|
@ -34,18 +34,19 @@ from reference_dict import my_aa_dict # CHECK DIR STRUC THERE!
|
|||
#=======================================================================
|
||||
#%% command line args
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
arg_parser.add_argument('-d', '--drug', help = 'drug name', default = None)
|
||||
arg_parser.add_argument('-g', '--gene', help = 'gene name', default = None) # case sensitive
|
||||
arg_parser.add_argument('-d', '--drug', help = 'drug name', default = 'pyrazinamide')
|
||||
arg_parser.add_argument('-g', '--gene', help = 'gene name', default = 'pncA') # case sensitive
|
||||
args = arg_parser.parse_args()
|
||||
#=======================================================================
|
||||
#%% variable assignment: input and output
|
||||
drug = 'pyrazinamide'
|
||||
gene = 'pncA'
|
||||
gene_match = gene + '_p.'
|
||||
#drug = 'pyrazinamide'
|
||||
#gene = 'pncA'
|
||||
#gene_match = gene + '_p.'
|
||||
|
||||
# cmd variables
|
||||
#drug = args.drug
|
||||
#gene = args.gene
|
||||
drug = args.drug
|
||||
gene = args.gene
|
||||
gene_match = gene + '_p.'
|
||||
|
||||
#==========
|
||||
# dir
|
||||
|
|
@ -57,9 +58,7 @@ outdir = datadir + '/' + drug + '/' + 'output'
|
|||
# input
|
||||
#=======
|
||||
in_filename_afor = gene.lower() + '_af_or.csv'
|
||||
# FIXME
|
||||
in_filename_afor_kin = gene.lower() + '_af_or_kinship.csv'
|
||||
# needs to contain OR. it only has beta!
|
||||
|
||||
infile1 = outdir + '/' + in_filename_afor
|
||||
infile2 = outdir + '/' + in_filename_afor_kin
|
||||
|
|
@ -78,7 +77,7 @@ print('Output file:', outfile
|
|||
, '\n===================================================================')
|
||||
|
||||
|
||||
del(in_filename_afor, in_filename_afor_kin, outfile, datadir, outdir)
|
||||
del(in_filename_afor, in_filename_afor_kin, datadir, outdir)
|
||||
#%% end of variable assignment for input and output files
|
||||
#=======================================================================
|
||||
#%% format mutations
|
||||
|
|
@ -169,12 +168,14 @@ print(ndiff1)
|
|||
ndiff2 = afor_kin_df_nrows - afor_kin_df['mutationinformation'].isin(afor_df['mutationinformation']).sum()
|
||||
print(ndiff2)
|
||||
|
||||
#%% combining dfs
|
||||
|
||||
# Define join type
|
||||
#my_join = 'inner'
|
||||
#my_join = 'right'
|
||||
##my_join = 'left'
|
||||
#my_join = 'left'
|
||||
my_join = 'outer'
|
||||
|
||||
fail = False
|
||||
# sanity check: how many muts from afor_kin_df are in afor_df. should be a complete subset
|
||||
if ndiff2 == 0:
|
||||
print('PASS: all muts in afor_kin_df are present in afor_df'
|
||||
|
|
@ -182,64 +183,31 @@ if ndiff2 == 0:
|
|||
|
||||
combined_df = pd.merge(afor_df, afor_kin_df, on = merging_cols, how = my_join)
|
||||
|
||||
if my_join == 'outer':
|
||||
if my_join == ('outer' or 'left') :
|
||||
print('combing with:', my_join)
|
||||
expected_rows = afor_df_nrows + ndiff1
|
||||
expected_cols = (afor_df_ncols + afor_kin_df_ncols) - ncommon_cols
|
||||
if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols:
|
||||
print('PASS: successfully combined dfs with:', my_join, 'join')
|
||||
else:
|
||||
print('FAIL: ', my_join, 'join')
|
||||
|
||||
print('\nExpected no. of rows:', expected_rows
|
||||
, '\nGot:', len(combined_df)
|
||||
, '\nExpected no. of cols:', expected_cols
|
||||
, '\nGot:', len(combined_df.columns))
|
||||
|
||||
elif my_join == 'inner':
|
||||
if my_join == ('inner' or 'right'):
|
||||
print('combing with:', my_join)
|
||||
expected_rows = afor_kin_df_nrows
|
||||
expected_cols = (afor_df_ncols + afor_kin_df_ncols) - ncommon_cols
|
||||
if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols:
|
||||
print('PASS: successfully combined dfs with:', my_join, 'join')
|
||||
else:
|
||||
print('FAIL: ', my_join, 'join')
|
||||
|
||||
print('\nExpected no. of rows:', expected_rows
|
||||
, '\nGot:', len(combined_df)
|
||||
, '\nExpected no. of cols:', expected_cols
|
||||
, '\nGot:', len(combined_df.columns))
|
||||
|
||||
elif my_join == 'left':
|
||||
expected_rows = afor_df_nrows + ndiff1
|
||||
expected_cols = (afor_df_ncols + afor_kin_df_ncols) - ncommon_cols
|
||||
if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols:
|
||||
print('PASS: successfully combined dfs with:', my_join, 'join')
|
||||
else:
|
||||
print('FAIL: ', my_join, 'join')
|
||||
|
||||
print('\nExpected no. of rows:', expected_rows
|
||||
, '\nGot:', len(combined_df)
|
||||
, '\nExpected no. of cols:', expected_cols
|
||||
, '\nGot:', len(combined_df.columns))
|
||||
|
||||
|
||||
elif my_join == 'right':
|
||||
expected_rows = afor_kin_df_nrows
|
||||
expected_cols = (afor_df_ncols + afor_kin_df_ncols) - ncommon_cols
|
||||
if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols:
|
||||
print('PASS: successfully combined dfs with:', my_join, 'join')
|
||||
else:
|
||||
print('FAIL: ', my_join, 'join')
|
||||
|
||||
print('\nExpected no. of rows:', expected_rows
|
||||
, '\nGot:', len(combined_df)
|
||||
, '\nExpected no. of cols:', expected_cols
|
||||
, '\nGot:', len(combined_df.columns))
|
||||
|
||||
expected_cols = afor_df_ncols + afor_kin_df_ncols - ncommon_cols
|
||||
|
||||
if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols:
|
||||
print('PASS: successfully combined dfs with:', my_join, 'join')
|
||||
else:
|
||||
print('FAIL: failed to combine dfs, expected rows and cols not matched')
|
||||
print('FAIL: combined_df\'s expected rows and cols not matched')
|
||||
fail = True # BAD practice! just a placeholder to avoid code duplication
|
||||
|
||||
print('\nExpected no. of rows:', expected_rows
|
||||
, '\nGot:', len(combined_df)
|
||||
, '\nExpected no. of cols:', expected_cols
|
||||
, '\nGot:', len(combined_df.columns))
|
||||
if fail:
|
||||
sys.exit('ERROR: combined_df may be incorrectly combined')
|
||||
else:
|
||||
print('FAIL: numbers mismatch, mutations present in afor_kin_df but not in afor_df')
|
||||
|
||||
sys.exit('ERROR: Not all mutations in the kinship_df are present in the df with other ORs')
|
||||
|
||||
#%% check duplicate cols: ones containing suffix '_x' or '_y'
|
||||
# should only be position
|
||||
|
|
@ -256,78 +224,73 @@ combined_or_df['position'].head()
|
|||
|
||||
# recheck
|
||||
foo = combined_or_df.filter(regex = r'.*_x|_y', axis = 1)
|
||||
print(foo.columns) # should only be position
|
||||
print(foo.columns) # should only be empty
|
||||
|
||||
combined_or_df['af'].head()
|
||||
combined_or_df.rename(columns = {'af': 'af_kin'}, inplace = True)
|
||||
combined_or_df['af_kin']
|
||||
#%% calculate OR for kinship
|
||||
combined_or_df['or_kin'] = np.exp(combined_or_df['beta'])
|
||||
#%% rearraging columns
|
||||
print('Dim of df prefromatting:', combined_or_df.shape)
|
||||
|
||||
# drop duplicate columns
|
||||
#if combined_or_df['alternate_allele'].equals(combined_or_df['alt_allele0']):
|
||||
# combined_or_df.drop('alternate_allele', axis = 1, inplace = True)
|
||||
|
||||
combined_or_df2 = combined_or_df.T.drop_duplicates().T# changes dtypes in cols
|
||||
dup_cols = set(combined_or_df.columns).difference(combined_or_df2.columns)
|
||||
#tot_diff is equal to n_diff
|
||||
|
||||
# drop some not required cols
|
||||
combined_or_df.drop(list(dup_cols), axis = 1, inplace = True)
|
||||
print(combined_or_df.columns)
|
||||
combined_or_df.drop(['chromosome_text', 'chr', 'symbol', '_merge', ], axis = 1, inplace = True)
|
||||
|
||||
combined_or_df.rename(columns = {'ref_allele1': 'reference_allele'}, inplace = True)
|
||||
|
||||
print(combined_or_df.columns)
|
||||
|
||||
|
||||
#%% reorder columns
|
||||
#https://stackoverflow.com/questions/13148429/how-to-change-the-order-of-dataframe-columns
|
||||
|
||||
# check af: curiosity
|
||||
|
||||
# setting column's order
|
||||
output_df = combined_or_df[['mutation', 'wild_type', 'position', 'mutant_type', 'mutationinformation'
|
||||
, 'chr_num_allele', 'ref_allele', 'alt_allele'
|
||||
, 'mut_info', 'mut_type', 'gene_id', 'gene_number', 'mut_region'
|
||||
, 'reference_allele', 'alternate_allele', 'chromosome_number'
|
||||
, 'afs', 'af_kin', 'ors_logistic', 'ors_chi_cus', 'or_kin', 'ors_fisher'
|
||||
, 'pvals_logistic', 'pvals_fisher', 'p_wald', 'ci_lb_fisher', 'ci_ub_fisher'
|
||||
, 'beta', 'se', 'logl_H1', 'l_remle','stat_chi', 'pvals_chi', 'n_diff' , 'n_miss']]
|
||||
output_df = combined_or_df[['mutation',
|
||||
'mutationinformation',
|
||||
'wild_type',
|
||||
'position',
|
||||
'mutant_type',
|
||||
'chr_num_allele',
|
||||
'ref_allele',
|
||||
'alt_allele',
|
||||
'mut_info',
|
||||
'mut_type',
|
||||
'gene_id',
|
||||
'gene_number',
|
||||
'mut_region',
|
||||
'reference_allele',
|
||||
'alternate_allele',
|
||||
'chromosome_number',
|
||||
'af',
|
||||
'af_kin',
|
||||
'or_kin',
|
||||
'or_logistic',
|
||||
'or_mychisq',
|
||||
'est_chisq',
|
||||
'or_fisher',
|
||||
'ci_low_logistic',
|
||||
'ci_hi_logistic',
|
||||
'ci_low_fisher',
|
||||
'ci_hi_fisher',
|
||||
'pwald_kin',
|
||||
'pval_logistic',
|
||||
'pval_fisher',
|
||||
'pval_chisq',
|
||||
'beta_logistic',
|
||||
'beta_kin',
|
||||
'se_logistic',
|
||||
'se_kin',
|
||||
'zval_logistic',
|
||||
'logl_H1_kin',
|
||||
'l_remle_kin',
|
||||
'n_diff',
|
||||
'tot_diff',
|
||||
'n_miss']]
|
||||
|
||||
#%% output combined or df
|
||||
#===============
|
||||
# writing file
|
||||
#===============
|
||||
print('Writing file...')
|
||||
|
||||
#combined_or_df.to_csv(outfile, header = True, index = False)
|
||||
output_df.to_csv(outfile, header = True, index = False)
|
||||
|
||||
print('Finished writing file:', outfile
|
||||
, '\nNo. of rows:', len(combined_or_df)
|
||||
, '\nNo. of cols:', len(combined_or_df.columns)
|
||||
, '\n=========================================================')
|
||||
|
||||
#%% practice
|
||||
df = pd.DataFrame()
|
||||
column_names = ['x','y','z','mean']
|
||||
for col in column_names:
|
||||
df[col] = np.random.randint(0,100, size=10000)
|
||||
df.head()
|
||||
|
||||
# drop duplicate col with dup values not necessarily colnames
|
||||
df['xdup'] = df['x']
|
||||
df
|
||||
|
||||
df = df.T.drop_duplicates().T
|
||||
|
||||
#import math
|
||||
math.exp(0)
|
||||
|
||||
df['expX'] = np.exp(df['x']) # math doesn't understand series dtype
|
||||
df
|
||||
#%%
|
||||
# sanity check after rearranging
|
||||
|
||||
if combined_or_df.shape == output_df.shape and set(combined_or_df.columns) == set(output_df.columns):
|
||||
print('PASS: Successfully formatted df with rearranged columns')
|
||||
else:
|
||||
sys.exit('FAIL: something went wrong when rearranging columns!')
|
||||
|
||||
#%% write file
|
||||
print('\n====================================================================='
|
||||
, '\nWriting output file:\n', outfile
|
||||
, '\nNo.of rows:', len(output_df)
|
||||
, '\nNo. of cols:', len(output_df.columns))
|
||||
output_df.to_csv(outfile, index = False)
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue