renamed files that combine dfs
This commit is contained in:
parent
a220288c5f
commit
5addb85851
5 changed files with 187 additions and 622 deletions
|
|
@ -1,393 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
Created on Tue Aug 6 12:56:03 2019
|
||||
|
||||
@author: tanu
|
||||
'''
|
||||
# FIXME: change filename 4 (mcsm normalised data)
|
||||
# to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline
|
||||
#=======================================================================
|
||||
# Task: combine 2 dfs with aa position as linking column
|
||||
# This is done in 2 steps:
|
||||
# merge 1:
|
||||
|
||||
# useful link
|
||||
# https://stackoverflow.com/questions/23668427/pandas-three-way-joining-multiple-dataframes-on-columns
|
||||
#=======================================================================
|
||||
#%% load packages
|
||||
import sys, os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import argparse
|
||||
#=======================================================================
|
||||
#%% specify input and curr dir
|
||||
homedir = os.path.expanduser('~')
|
||||
|
||||
# set working dir
|
||||
os.getcwd()
|
||||
os.chdir(homedir + '/git/LSHTM_analysis/scripts')
|
||||
os.getcwd()
|
||||
|
||||
# local import
|
||||
#from reference_dict import my_aa_dict # CHECK DIR STRUC THERE!
|
||||
from reference_dict import low_3letter_dict
|
||||
#=======================================================================
|
||||
#%% command line args
|
||||
#arg_parser = argparse.ArgumentParser()
|
||||
#arg_parser.add_argument('-d', '--drug', help = 'drug name', default = 'pyrazinamide')
|
||||
#arg_parser.add_argument('-g', '--gene', help = 'gene name', default = 'pncA') # case sensitive
|
||||
#args = arg_parser.parse_args()
|
||||
#=======================================================================
|
||||
#%% variable assignment: input and output
|
||||
drug = 'pyrazinamide'
|
||||
gene = 'pncA'
|
||||
gene_match = gene + '_p.'
|
||||
|
||||
# cmd variables
|
||||
#drug = args.drug
|
||||
#gene = args.gene
|
||||
#gene_match = gene + '_p.'
|
||||
|
||||
#==========
|
||||
# dir
|
||||
#==========
|
||||
datadir = homedir + '/' + 'git/Data'
|
||||
indir = datadir + '/' + drug + '/' + 'input'
|
||||
outdir = datadir + '/' + drug + '/' + 'output'
|
||||
|
||||
#=======
|
||||
# input
|
||||
#=======
|
||||
in_filename_snpinfo = 'ns' + gene.lower() + '_snp_info.csv'
|
||||
in_filename_afor = gene.lower() + '_af_or.csv'
|
||||
in_filename_afor_kin = gene.lower() + '_af_or_kinship.csv'
|
||||
|
||||
infile0 = indir + '/' + in_filename_snpinfo
|
||||
infile1 = outdir + '/' + in_filename_afor
|
||||
infile2 = outdir + '/' + in_filename_afor_kin
|
||||
|
||||
|
||||
print('Input file0:', infile0
|
||||
, '\nInput file1:', infile1
|
||||
, '\nInput file2:', infile2
|
||||
, '\n=============================================================')
|
||||
|
||||
#=======
|
||||
# output
|
||||
#=======
|
||||
out_filename = gene.lower() + '_metadata_afs_ors.csv'
|
||||
outfile = outdir + '/' + out_filename
|
||||
print('Output file:', outfile
|
||||
, '\n=============================================================')
|
||||
|
||||
|
||||
del(in_filename_afor, in_filename_afor_kin, datadir, indir, outdir)
|
||||
#%% end of variable assignment for input and output files
|
||||
#=======================================================================
|
||||
#%% format mutations
|
||||
# mut_format: gene.abc1cde | 1A>1B
|
||||
|
||||
#========================
|
||||
# read input csv files to combine
|
||||
#========================
|
||||
snpinfo_df = pd.read_csv(infile0, sep = ',')
|
||||
#snpinfo_ncols = len(snpinfo_df.columns)
|
||||
#snpinfo.shape[0] = len(snpinfo_df)
|
||||
print('No. of rows in', infile0, ':', snpinfo_df.shape[0]
|
||||
, '\nNo. of cols in', infile0, ':', snpinfo_df.shape[1])
|
||||
|
||||
afor_df = pd.read_csv(infile1, sep = ',')
|
||||
#afor_ncols = len(afor_df.columns)
|
||||
#afor.shape[0] = len(afor_df)
|
||||
print('No. of rows in', infile1, ':', afor_df.shape[0]
|
||||
, '\nNo. of cols in', infile1, ':', afor_df.shape[1])
|
||||
|
||||
afor_kin_df = pd.read_csv(infile2, sep = ',')
|
||||
#afor_kin.shape[0] = len(afor_kin_df)
|
||||
#afor_kin_ncols = len(afor_kin_df.columns)
|
||||
print('No. of rows in', infile2, ':', afor_kin_df.shape[0]
|
||||
, '\nNo. of cols in', infile2, ':', afor_kin_df.shape[1])
|
||||
|
||||
#%% Process afor_df
|
||||
#1) pull all snp_info so you have ref_allele, etc
|
||||
# i.e merge afor_df and snpinfo_df
|
||||
# find merging column
|
||||
|
||||
left_df = afor_df.copy()
|
||||
right_df = snpinfo_df.copy()
|
||||
|
||||
common_cols = np.intersect1d(left_df.columns, right_df.columns).tolist()
|
||||
print('Length of common cols:', len(common_cols)
|
||||
, '\ncommon column/s:', common_cols, 'type:', type(common_cols))
|
||||
|
||||
#https://stackoverflow.com/questions/44639772/python-pandas-column-dtype-object-causing-merge-to-fail-with-dtypewarning-colu
|
||||
print('selecting consistent dtypes for merging (object i.e string)')
|
||||
merging_cols = left_df[common_cols].select_dtypes(include = [object]).columns.tolist()
|
||||
print(merging_cols)
|
||||
nmerging_cols = len(merging_cols)
|
||||
print(' length of merging cols:', nmerging_cols
|
||||
, '\nmerging cols:', merging_cols, 'type:', type(merging_cols))
|
||||
|
||||
#https://stackoverflow.com/questions/22720739/pandas-left-outer-join-results-in-table-larger-than-left-table
|
||||
# drop duplicates else the expected rows don't match
|
||||
print('Checking for duplicates in common col:', common_cols
|
||||
, '\nNo of duplicates:'
|
||||
, len(right_df[right_df.duplicated(common_cols)])
|
||||
, '\noriginal length:', right_df.shape[0])
|
||||
|
||||
right_df = right_df[~right_df.duplicated(common_cols)]
|
||||
print('\nrevised length:', right_df.shape[0])
|
||||
|
||||
# checking cross-over of mutations in the two dfs to merge
|
||||
ndiff1 = left_df.shape[0] - left_df['mutation'].isin(right_df['mutation']).sum()
|
||||
print('There are', ndiff1, 'mutations with OR, but no snp_info'
|
||||
, '\nExtracting and writing out file')
|
||||
missing_mutinfo = left_df[~left_df['mutation'].isin(right_df['mutation'])]
|
||||
#missing_mutinfo.to_csv('infoless_muts.csv')
|
||||
|
||||
ndiff2 = right_df.shape[0] - right_df['mutation'].isin(left_df['mutation']).sum()
|
||||
print('There are', ndiff2, 'mutations that do not have OR, but have snp_info')
|
||||
|
||||
# Define join type
|
||||
#my_join = 'inner'
|
||||
#my_join = 'outer'
|
||||
#my_join = 'right'
|
||||
my_join = 'left'
|
||||
|
||||
print('combing with join:', my_join)
|
||||
combined_df1 = pd.merge(left_df, right_df, on = merging_cols, how = my_join)
|
||||
print('\nshape:', combined_df1.shape)
|
||||
|
||||
# inner = 252
|
||||
left_df.shape[0] - ndiff1
|
||||
|
||||
# outer = 331
|
||||
right_df.shape[0] + ndiff1
|
||||
|
||||
# right = 290
|
||||
right_df.shape[0]
|
||||
|
||||
# left = 293
|
||||
left_df.shape[0]
|
||||
|
||||
|
||||
#%%
|
||||
# see if you want an extra clause here!
|
||||
# Define join type
|
||||
#my_join = 'inner'
|
||||
#my_join = 'outer'
|
||||
#my_join = 'right'
|
||||
my_join = 'left'
|
||||
|
||||
fail = False
|
||||
print('combing with:', my_join)
|
||||
combined_df1 = pd.merge(left_df, right_df, on = merging_cols, how = my_join)
|
||||
|
||||
if my_join == 'inner':
|
||||
#expected_rows = left_df.shape[0] - ndiff1
|
||||
expected_rows = left_df.shape[0] - ndiff1
|
||||
|
||||
if my_join == 'outer':
|
||||
#expected_rows = right_df.shape[0] + ndiff1
|
||||
expected_rows = right_df.shape[0] + ndiff1
|
||||
|
||||
if my_join == 'right':
|
||||
#expected_rows = right_df.shape[0]
|
||||
expected_rows = right_df.shape[0]
|
||||
|
||||
if my_join == 'left':
|
||||
#expected_rows = left_df.shape[0]
|
||||
expected_rows = left_df.shape[0]
|
||||
|
||||
expected_cols = left_df.shape[1] + right_df.shape[1] - nmerging_cols
|
||||
|
||||
if len(combined_df1) == expected_rows and len(combined_df1.columns) == expected_cols:
|
||||
print('PASS: successfully combined dfs with:', my_join, 'join')
|
||||
else:
|
||||
print('FAIL: combined_df\'s expected rows and cols not matched')
|
||||
fail = True
|
||||
print('\nExpected no. of rows:', expected_rows
|
||||
, '\nGot:', len(combined_df1)
|
||||
, '\nExpected no. of cols:', expected_cols
|
||||
, '\nGot:', len(combined_df1.columns))
|
||||
if fail:
|
||||
sys.exit()
|
||||
|
||||
# delete variables
|
||||
del(left_df, right_df, common_cols, merging_cols, nmerging_cols, my_join, ndiff1, ndiff2, missing_mutinfo
|
||||
, expected_rows, expected_cols, fail)
|
||||
del(afor_df, snpinfo_df)
|
||||
#=======================================================================
|
||||
|
||||
#%% Second merge: combined_df1 and afor_kin_df
|
||||
|
||||
left_df = combined_df1.copy()
|
||||
right_df = afor_kin_df.copy()
|
||||
|
||||
common_cols = np.intersect1d(left_df.columns, right_df.columns).tolist()
|
||||
print('Length of common cols:', len(common_cols)
|
||||
, '\ncommon column/s:', common_cols, 'type:', type(common_cols))
|
||||
|
||||
#https://stackoverflow.com/questions/44639772/python-pandas-column-dtype-object-causing-merge-to-fail-with-dtypewarning-colu
|
||||
print('selecting consistent dtypes for merging (object i.e string)')
|
||||
|
||||
#FIXME
|
||||
|
||||
#merging_cols = left_df[common_cols].select_dtypes(include = [object]).columns.tolist()
|
||||
merging_cols = ['wild_type', 'mutant_type', 'mutationinformation']
|
||||
nmerging_cols_cols = len(merging_cols)
|
||||
|
||||
print(merging_cols)
|
||||
nmerging_cols = len(merging_cols)
|
||||
print(' length of merging cols:', nmerging_cols
|
||||
, '\nmerging cols:', merging_cols, 'type:', type(merging_cols))
|
||||
|
||||
ndiff1 = left_df.shape[0] - left_df['mutationinformation'].isin(right_df['mutationinformation']).sum()
|
||||
print('There are', ndiff1, 'mutations with OR, but not in OR kinship'
|
||||
, '\nExtracting and writing out file')
|
||||
missing_mutinfo = left_df[~left_df['mutationinformation'].isin(right_df['mutationinformation'])]
|
||||
#missing_mutinfo.to_csv('infoless_muts.csv')
|
||||
|
||||
ndiff2 = right_df.shape[0] - right_df['mutationinformation'].isin(left_df['mutationinformation']).sum()
|
||||
print('There are', ndiff2, 'mutations that do not have OR, but have OR kinship')
|
||||
|
||||
my_join = 'outer'
|
||||
|
||||
fail = False
|
||||
print('combing with:', my_join)
|
||||
combined_df2 = pd.merge(left_df, right_df, on = merging_cols, how = my_join)
|
||||
|
||||
if my_join == 'inner':
|
||||
#expected_rows = left_df.shape[0] - ndiff1
|
||||
expected_rows = left_df.shape[0] - ndiff1
|
||||
|
||||
if my_join == 'outer':
|
||||
#expected_rows = right_df.shape[0] + ndiff1
|
||||
expected_rows = right_df.shape[0] + ndiff1
|
||||
|
||||
if my_join == 'right':
|
||||
#expected_rows = right_df.shape[0]
|
||||
expected_rows = right_df.shape[0]
|
||||
|
||||
if my_join == 'left':
|
||||
#expected_rows = left_df.shape[0]
|
||||
expected_rows = left_df.shape[0]
|
||||
|
||||
expected_cols = left_df.shape[1] + right_df.shape[1] - nmerging_cols
|
||||
|
||||
if len(combined_df2) == expected_rows and len(combined_df2.columns) == expected_cols:
|
||||
print('PASS: successfully combined dfs with:', my_join, 'join')
|
||||
else:
|
||||
print('FAIL: combined_df\'s expected rows and cols not matched')
|
||||
fail = True
|
||||
print('\nExpected no. of rows:', expected_rows
|
||||
, '\nGot:', len(combined_df2)
|
||||
, '\nExpected no. of cols:', expected_cols
|
||||
, '\nGot:', len(combined_df2.columns))
|
||||
if fail:
|
||||
sys.exit()
|
||||
#%% check duplicate cols: ones containing suffix '_x' or '_y'
|
||||
# should only be position
|
||||
foo = combined_df2.filter(regex = r'.*_x|_y', axis = 1)
|
||||
print(foo.columns) # should only be position
|
||||
|
||||
# drop position col containing suffix '_y' and then rename col without suffix
|
||||
combined_or_df = combined_df2.drop(combined_df2.filter(regex = r'.*_y').columns, axis = 1)
|
||||
#combined_or_df['position_x'].head()
|
||||
|
||||
# renaming columns
|
||||
#combined_or_df.rename(columns = {'position_x': 'position'}, inplace = True)
|
||||
#combined_or_df['position'].head()
|
||||
#recheck
|
||||
#foo = combined_or_df.filter(regex = r'.*_x|_y', axis = 1)
|
||||
#print(foo.columns) # should only be empty
|
||||
|
||||
|
||||
# remove '_x' from some cols
|
||||
|
||||
import re
|
||||
def clean_colnames(colname):
|
||||
|
||||
if re.search('.*_x', colname):
|
||||
pos = re.search('.*_x', colname).start()
|
||||
return colname[:pos]
|
||||
else:
|
||||
return colname
|
||||
|
||||
#https://stackoverflow.com/questions/26500156/renaming-column-in-dataframe-for-pandas-using-regular-expression
|
||||
combined_or_df.columns
|
||||
combined_or_df.rename(columns=lambda x: re.sub('_x$','',x), inplace = True)
|
||||
combined_or_df.columns
|
||||
|
||||
#FIXME: this should be 0 when you run the 35k dataset
|
||||
combined_or_df['chromosome_number'].isna().sum()
|
||||
|
||||
#%% rearraging columns
|
||||
print('Dim of df prefromatting:', combined_or_df.shape)
|
||||
|
||||
print(combined_or_df.columns, '\nshape:', combined_or_df.shape)
|
||||
|
||||
# removing unnecessary column
|
||||
combined_or_df = combined_or_df.drop(['symbol'], axis = 1)
|
||||
print(combined_or_df.columns, '\nshape:', combined_or_df.shape)
|
||||
#%% reorder columns
|
||||
#https://stackoverflow.com/questions/13148429/how-to-change-the-order-of-dataframe-columns
|
||||
# setting column's order
|
||||
output_df = combined_or_df[['mutation',
|
||||
'mutationinformation',
|
||||
'wild_type',
|
||||
'position',
|
||||
'mutant_type',
|
||||
'chr_num_allele',
|
||||
'ref_allele',
|
||||
'alt_allele',
|
||||
'mut_info',
|
||||
'mut_type',
|
||||
'gene_id',
|
||||
'gene_number',
|
||||
'mut_region',
|
||||
'reference_allele',
|
||||
'alternate_allele',
|
||||
'chromosome_number',
|
||||
'af',
|
||||
'af_kin',
|
||||
'or_kin',
|
||||
'or_logistic',
|
||||
'or_mychisq',
|
||||
'est_chisq',
|
||||
'or_fisher',
|
||||
'ci_low_logistic',
|
||||
'ci_hi_logistic',
|
||||
'ci_low_fisher',
|
||||
'ci_hi_fisher',
|
||||
'pwald_kin',
|
||||
'pval_logistic',
|
||||
'pval_fisher',
|
||||
'pval_chisq',
|
||||
'beta_logistic',
|
||||
'beta_kin',
|
||||
'se_logistic',
|
||||
'se_kin',
|
||||
'zval_logistic',
|
||||
'logl_H1_kin',
|
||||
'l_remle_kin',
|
||||
'wt_3let',
|
||||
'mt_3let',
|
||||
'n_diff',
|
||||
'tot_diff',
|
||||
'n_miss']]
|
||||
|
||||
# sanity check after rearranging
|
||||
if combined_or_df.shape == output_df.shape and set(combined_or_df.columns) == set(output_df.columns):
|
||||
print('PASS: Successfully formatted df with rearranged columns')
|
||||
else:
|
||||
sys.exit('FAIL: something went wrong when rearranging columns!')
|
||||
|
||||
#%% write file
|
||||
print('\n====================================================================='
|
||||
, '\nWriting output file:\n', outfile
|
||||
, '\nNo.of rows:', len(output_df)
|
||||
, '\nNo. of cols:', len(output_df.columns))
|
||||
output_df.to_csv(outfile, index = False)
|
||||
|
||||
|
|
@ -37,15 +37,15 @@ def detect_common_cols (df1, df2):
|
|||
@type: list
|
||||
"""
|
||||
common_cols = np.intersect1d(df1.columns, df2.columns).tolist()
|
||||
#print('Length of comm_cols:', len(comm_cols)
|
||||
# , '\nmerging column/s:', comm_cols
|
||||
# , '\ntype:', type(comm_cols)
|
||||
# , '\ndtypes in merging columns:\n', df1[comm_cols].dtypes)
|
||||
print('Length of comm_cols:', len(common_cols)
|
||||
, '\nmerging column/s:', common_cols
|
||||
, '\ntype:', type(common_cols)
|
||||
, '\ndtypes in merging columns:\n', df1[common_cols].dtypes)
|
||||
|
||||
return common_cols
|
||||
|
||||
|
||||
def combine_stability_dfs(df1, df2, my_join = 'outer'):
|
||||
def combine_dfs_with_checks(df1, df2, my_join = 'outer'):
|
||||
"""
|
||||
Combine 2 dfs by finding merging columns automatically
|
||||
|
||||
|
|
@ -62,14 +62,15 @@ def combine_stability_dfs(df1, df2, my_join = 'outer'):
|
|||
@type: pandas df
|
||||
"""
|
||||
|
||||
print('Finding comm_valson cols and merging cols:'
|
||||
print('Finding comm_cols and merging cols:'
|
||||
,'\n=========================================================')
|
||||
|
||||
common_cols = np.intersect1d(df1.columns, df2.columns).tolist()
|
||||
print('Length of comm_valson cols:', len(common_cols)
|
||||
print('Length of comm_cols:', len(common_cols)
|
||||
, '\nmerging column/s:', common_cols
|
||||
, '\ntype:', type(common_cols)
|
||||
, '\ndtypes in merging columns:\n', df1[common_cols].dtypes)
|
||||
, '\ntype:', type(common_cols))
|
||||
|
||||
#print('\ndtypes in merging columns:\n', df1[common_cols].dtypes)
|
||||
|
||||
print('selecting consistent dtypes for merging (object i.e string)')
|
||||
#merging_cols = df1[comm_valson_cols].select_dtypes(include = [object]).columns.tolist()
|
||||
|
|
@ -108,8 +109,7 @@ def combine_stability_dfs(df1, df2, my_join = 'outer'):
|
|||
fail = False
|
||||
print('combing with:', my_join)
|
||||
comb_df = pd.merge(df1, df2, on = merging_cols, how = my_join)
|
||||
combined_df = comb_df.drop_duplicates(subset = merging_cols, keep ='first')
|
||||
|
||||
|
||||
expected_cols = df1.shape[1] + df2.shape[1] - nmerging_cols
|
||||
|
||||
|
||||
|
|
@ -130,18 +130,16 @@ def combine_stability_dfs(df1, df2, my_join = 'outer'):
|
|||
# expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - comm_vals_count
|
||||
|
||||
|
||||
if my_join == 'inner' or 'outer' and len(merging_cols)>1:
|
||||
comm_vals = np.intersect1d(df1['mutationinformation'], df2['mutationinformation'])
|
||||
print('length of comm_values for merge:', len(comm_vals))
|
||||
if my_join == 'inner':
|
||||
expected_rows = len(comm_vals)
|
||||
if my_join == 'outer':
|
||||
df1_nd = df1.drop_duplicates(merging_cols, keep = 'first')
|
||||
df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
|
||||
expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - len(comm_vals)
|
||||
if my_join == ('inner' or 'outer') and len(merging_cols) > 1:
|
||||
#comm_vals = np.intersect1d(df1['mutationinformation'], df2['mutationinformation'])
|
||||
print('length of merging_cols > 1, therefore omitting row checks')
|
||||
combined_df = comb_df.copy()
|
||||
expected_rows = len(combined_df)
|
||||
|
||||
else:
|
||||
comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols])
|
||||
print('length of comm_values for merge:', len(comm_vals))
|
||||
print('length of merging_cols == 1, calculating expected rows in merged_df')
|
||||
combined_df = comb_df.drop_duplicates(subset = merging_cols, keep ='first')
|
||||
if my_join == 'inner':
|
||||
expected_rows = len(comm_vals)
|
||||
if my_join == 'outer':
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
Created on Tue Aug 6 12:56:03 2019
|
||||
|
||||
@author: tanu
|
||||
'''
|
||||
# FIXME: change filename 2(mcsm normalised data)
|
||||
# to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline
|
||||
#=======================================================================
|
||||
# Task: combine 2 dfs with aa position as linking column
|
||||
|
||||
# Input: 2 dfs
|
||||
# <gene.lower()>_complex_mcsm_norm.csv
|
||||
# <gene.lower()>_foldx.csv
|
||||
|
||||
# Output: .csv of all 2 dfs combined
|
||||
|
||||
# useful link
|
||||
# https://stackoverflow.com/questions/23668427/pandas-three-way-joining-multiple-dataframes-on-columns
|
||||
#=======================================================================
|
||||
#%% load packages
|
||||
import sys, os
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
#from varname import nameof
|
||||
import argparse
|
||||
from combining import combine_stability_dfs
|
||||
#=======================================================================
|
||||
#%% specify input and curr dir
|
||||
homedir = os.path.expanduser('~')
|
||||
|
||||
# set working dir
|
||||
os.getcwd()
|
||||
os.chdir(homedir + '/git/LSHTM_analysis/scripts')
|
||||
os.getcwd()
|
||||
#=======================================================================
|
||||
#%% command line args
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
arg_parser.add_argument('-d', '--drug', help='drug name', default = 'pyrazinamide')
|
||||
arg_parser.add_argument('-g', '--gene', help='gene name', default = 'pncA') # case sensitive
|
||||
args = arg_parser.parse_args()
|
||||
#=======================================================================
|
||||
#%% variable assignment: input and output
|
||||
#drug = 'pyrazinamide'
|
||||
#gene = 'pncA'
|
||||
#gene_match = gene + '_p.'
|
||||
|
||||
drug = args.drug
|
||||
gene = args.gene
|
||||
#======
|
||||
# dirs
|
||||
#======
|
||||
datadir = homedir + '/' + 'git/Data'
|
||||
indir = datadir + '/' + drug + '/' + 'input'
|
||||
outdir = datadir + '/' + drug + '/' + 'output'
|
||||
|
||||
#=======
|
||||
# input
|
||||
#=======
|
||||
in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv'
|
||||
in_filename_foldx = gene.lower() + '_foldx.csv'
|
||||
|
||||
infile_mcsm = outdir + '/' + in_filename_mcsm
|
||||
infile_foldx = outdir + '/' + in_filename_foldx
|
||||
|
||||
print('\nInput path:', outdir
|
||||
, '\nInput filename1:', in_filename_mcsm
|
||||
, '\nInput filename2:', in_filename_foldx
|
||||
, '\n============================================================')
|
||||
|
||||
#=======
|
||||
# output
|
||||
#=======
|
||||
out_filename_comb = gene.lower() + '_mcsm_foldx.csv'
|
||||
outfile_comb = outdir + '/' + out_filename_comb
|
||||
print('Output filename:', outfile_comb
|
||||
, '\n============================================================')
|
||||
|
||||
my_join_type = 'outer'
|
||||
#my_join_type = 'left'
|
||||
#my_join_type = 'right'
|
||||
#my_join_type = 'inner'
|
||||
|
||||
# end of variable assignment for input and output files
|
||||
#%% call function
|
||||
#=======================================================================
|
||||
#combine_stability_dfs(mcsm_df, foldx_df, outfile)
|
||||
#=======================================================================
|
||||
def main():
|
||||
|
||||
combined_df = combine_stability_dfs(infile_mcsm, infile_foldx, my_join = my_join_type)
|
||||
print('Combining 2 dfs...'
|
||||
, '\nArguments to function combine_stability_dfs:'
|
||||
, '\ndf1:', in_filename_mcsm
|
||||
, '\ndf2:', in_filename_foldx
|
||||
, '\njoin_type:', my_join_type
|
||||
, '\ncombined df sneak peak:\n'
|
||||
, combined_df.head())
|
||||
|
||||
print('Writing output...')
|
||||
|
||||
combined_df.to_csv(outfile_comb, index = False)
|
||||
|
||||
print('Finished writing output file'
|
||||
, '\nOutput file:', outfile_comb
|
||||
, '\nDimensions:', combined_df.shape)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
#=======================================================================
|
||||
#%% end of script
|
||||
|
|
@ -25,8 +25,7 @@ import pandas as pd
|
|||
import numpy as np
|
||||
#from varname import nameof
|
||||
import argparse
|
||||
from combining import combine_stability_dfs
|
||||
from combining import detect_common_cols
|
||||
|
||||
#=======================================================================
|
||||
#%% specify input and curr dir
|
||||
homedir = os.path.expanduser('~')
|
||||
|
|
@ -35,6 +34,10 @@ homedir = os.path.expanduser('~')
|
|||
os.getcwd()
|
||||
os.chdir(homedir + '/git/LSHTM_analysis/scripts')
|
||||
os.getcwd()
|
||||
|
||||
# local imports
|
||||
from combining_dfs import combine_dfs_with_checks
|
||||
from combining_dfs import detect_common_cols
|
||||
#=======================================================================
|
||||
#%% command line args
|
||||
#arg_parser = argparse.ArgumentParser()
|
||||
|
|
@ -60,33 +63,33 @@ outdir = datadir + '/' + drug + '/' + 'output'
|
|||
# input
|
||||
#=======
|
||||
#in_filename_linking = gene.lower() + '_linking_df.csv'
|
||||
#in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv'
|
||||
#in_filename_foldx = gene.lower() + '_foldx.csv'
|
||||
in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv'
|
||||
in_filename_foldx = gene.lower() + '_foldx.csv'
|
||||
in_filename_dssp = gene.lower() + '_dssp.csv'
|
||||
in_filename_kd = gene.lower() + '_kd.csv'
|
||||
#in_filename_rd = gene.lower() + '_rd.csv'
|
||||
in_filename_rd = gene.lower() + '_rd.csv'
|
||||
in_filename_snpinfo = 'ns' + gene.lower() + '_snp_info.csv'
|
||||
in_filename_afor = gene.lower() + '_af_or.csv'
|
||||
in_filename_afor_kin = gene.lower() + '_af_or_kinship.csv'
|
||||
|
||||
|
||||
#infile_linking = outdir + '/' + in_filename_linking
|
||||
#infile_mcsm = outdir + '/' + in_filename_mcsm
|
||||
#infile_foldx = outdir + '/' + in_filename_foldx
|
||||
infile_mcsm = outdir + '/' + in_filename_mcsm
|
||||
infile_foldx = outdir + '/' + in_filename_foldx
|
||||
infile_dssp = outdir + '/' + in_filename_dssp
|
||||
infile_kd = outdir + '/' + in_filename_kd
|
||||
#infile_rd = outdir + '/' + in_filename_rd
|
||||
infile_rd = outdir + '/' + in_filename_rd
|
||||
infile_snpinfo = indir + '/' + in_filename_snpinfo
|
||||
infile_afor = outdir + '/' + in_filename_afor
|
||||
infile_afor_kin = outdir + '/' + in_filename_afor_kin
|
||||
|
||||
|
||||
print('\nInput path:', outdir
|
||||
# , '\nInput filename1:', infile_mcsm
|
||||
# , '\nInput filename2:', infile_foldx
|
||||
, '\nInput filename2:', infile_dssp
|
||||
, '\nInput filename2:', infile_kd
|
||||
# , '\nInput filename2:', infile_rd
|
||||
, '\nInput filename mcsm:', infile_mcsm
|
||||
, '\nInput filename foldx:', infile_foldx
|
||||
, '\nInput filename dssp:', infile_dssp
|
||||
, '\nInput filename kd:', infile_kd
|
||||
, '\nInput filename rd', infile_rd
|
||||
, '\nInput filename snp info:', infile_snpinfo
|
||||
, '\nInput filename af or:', infile_afor
|
||||
, '\nInput filename afor kinship:', infile_afor_kin
|
||||
|
|
@ -95,10 +98,10 @@ print('\nInput path:', outdir
|
|||
#=======
|
||||
# output
|
||||
#=======
|
||||
#out_filename_comb = gene.lower() + '_struct_params_TEST.csv'
|
||||
#outfile_comb = outdir + '/' + out_filename_comb
|
||||
#print('Output filename:', outfile_comb
|
||||
# , '\n============================================================')
|
||||
out_filename_comb = gene.lower() + '_all_params.csv'
|
||||
outfile_comb = outdir + '/' + out_filename_comb
|
||||
print('Output filename:', outfile_comb
|
||||
, '\n============================================================')
|
||||
|
||||
o_join = 'outer'
|
||||
l_join = 'left'
|
||||
|
|
@ -111,6 +114,8 @@ i_join = 'inner'
|
|||
|
||||
#=======================================================================
|
||||
# call function to detect common cols
|
||||
# FIXME: do the OR combining in the end to iron out any problems
|
||||
# Couldn't run the function combin
|
||||
#=======================================================================
|
||||
def main():
|
||||
|
||||
|
|
@ -133,99 +138,166 @@ def main():
|
|||
# , '\nmerging column/s:', merging_cols, 'type:', type(merging_cols)
|
||||
# , '\ndtypes in merging columns:', dssp_df[merging_cols].dtypes)
|
||||
|
||||
#combined_df1 = combine_stability_dfs(dssp_df, kd_df, my_join = o_join)
|
||||
#combined_df1 = combine_dfs_with_checks(dssp_df, kd_df, my_join = o_join)
|
||||
#print('Dimensions of combined df:', combined_df1.shape
|
||||
# , '\nsneak peak:', combined_df1.head()
|
||||
# , '\ndtypes in cols:\n', combined_df1.dtypes)
|
||||
|
||||
#=============================================================================
|
||||
afor_df = pd.read_csv(infile_afor, sep = ',')
|
||||
afor_df.columns = afor_df.columns.str.lower()
|
||||
|
||||
snpinfo_df = pd.read_csv(infile_snpinfo, sep = ',')
|
||||
snpinfo_df.columns = snpinfo_df.columns.str.lower()
|
||||
|
||||
# print('Dimension df1:', afor_df.shape
|
||||
# , '\nDimension df2:', snpinfo_df.shape
|
||||
# , '\njoin type:', l_join
|
||||
# , '\n=========================================================')
|
||||
|
||||
|
||||
# detect common cols
|
||||
merging_cols = detect_common_cols(afor_df, snpinfo_df)
|
||||
#print('Length of common cols:', len(merging_cols)
|
||||
# , '\nmerging column/s:', merging_cols, 'type:', type(merging_cols)
|
||||
# , '\ndtypes in merging columns:', snpinfo_df[merging_cols].dtypes)
|
||||
|
||||
comb_afor_snpinfo = combine_stability_dfs(afor_df, snpinfo_df, my_join = l_join)
|
||||
#print('Dimensions of combined df:', comb_afor_snpinfo.shape
|
||||
# , '\nsneak peak:', comb_afor_snpinfo.head()
|
||||
# , '\ndtypes in cols:\n', comb_afor_snpinfo.dtypes)
|
||||
|
||||
#=============================================================================
|
||||
afor_kin_df = pd.read_csv(infile_afor_kin, sep = ',')
|
||||
afor_kin_df.columns = afor_kin_df.columns.str.lower()
|
||||
|
||||
# detect common cols
|
||||
merging_cols = detect_common_cols(comb_afor_snpinfo, afor_kin_df)
|
||||
|
||||
# comb2 = combine_stability_dfs(comb_afor_snpinfo, afor_kin_df, my_join = o_join)
|
||||
#print('Dimensions of combined df:', comb2.shape
|
||||
# , '\nsneak peak:', comb2.head()
|
||||
# , '\ndtypes in cols:\n', comb2.dtype)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
#if __name__ == '__main__':
|
||||
# main()
|
||||
#=======================================================================
|
||||
#%% end of script
|
||||
#hardocoded test
|
||||
#hardcoded test
|
||||
|
||||
mcsm_df = pd.read_csv(infile_mcsm, sep = ',')
|
||||
mcsm_df.columns = mcsm_df.columns.str.lower()
|
||||
foldx_df = pd.read_csv(infile_foldx , sep = ',')
|
||||
|
||||
print('==================================='
|
||||
, '\nFirst merge: mcsm + foldx'
|
||||
, '\n===================================')
|
||||
#mcsm_foldx_dfs = combine_dfs_with_checks(mcsm_df, foldx_df, my_join = o_join)
|
||||
merging_cols_m1 = detect_common_cols(mcsm_df, foldx_df)
|
||||
|
||||
mcsm_foldx_dfs = pd.merge(mcsm_df, foldx_df, on = merging_cols_m1, how = 'outer')
|
||||
ncols_m1 = len(mcsm_foldx_dfs.columns)
|
||||
|
||||
print('==================================='
|
||||
, '\nSecond merge: dssp + kd'
|
||||
, '\n===================================')
|
||||
|
||||
dssp_df = pd.read_csv(infile_dssp, sep = ',')
|
||||
kd_df = pd.read_csv(infile_kd, sep = ',')
|
||||
rd_df = pd.read_csv(infile_rd, sep = ',')
|
||||
|
||||
#dssp_kd_dfs = combine_dfs_with_checks(dssp_df, kd_df, my_join = o_join)
|
||||
merging_cols_m2 = detect_common_cols(dssp_df, kd_df)
|
||||
|
||||
dssp_kd_dfs = pd.merge(dssp_df, kd_df, on = merging_cols_m2, how = 'outer')
|
||||
|
||||
print('==================================='
|
||||
, '\nThird merge: dssp_kd_dfs + rd_df'
|
||||
, '\n===================================')
|
||||
#dssp_kd_rd_dfs = combine_dfs_with_checks(dssp_kd_dfs, rd_df, my_join = o_join)
|
||||
merging_cols_m3 = detect_common_cols(dssp_df, kd_df)
|
||||
dssp_kd_rd_dfs = pd.merge(dssp_kd_dfs, rd_df, on = merging_cols_m3, how = 'outer')
|
||||
|
||||
ncols_m3 = len(dssp_kd_rd_dfs.columns)
|
||||
|
||||
print('==================================='
|
||||
, '\nFourth merge: First merge + Third merge'
|
||||
, '\n===================================')
|
||||
#combined_dfs = combine_dfs_with_checks(mcsm_foldx_dfs, dssp_kd_rd_dfs, my_join = i_join)# gives wrong!
|
||||
merging_cols_m4 = detect_common_cols(mcsm_foldx_dfs, dssp_kd_rd_dfs)
|
||||
combined_df_expected_cols = ncols_m1 + ncols_m3 - len(merging_cols_m4)
|
||||
|
||||
combined_df = pd.merge(mcsm_foldx_dfs, dssp_kd_rd_dfs, on = merging_cols_m4, how = 'inner')
|
||||
|
||||
|
||||
if len(combined_df) == len(mcsm_df) and len(combined_df.columns) == combined_df_expected_cols:
|
||||
print('PASS: successfully combined 5 dfs'
|
||||
, '\nnrows combined_df:', len(combined_df)
|
||||
, '\ncols combined_df:', len(combined_df.columns))
|
||||
else:
|
||||
sys.exit('FAIL: check individual df merges')
|
||||
|
||||
#%% OR combining
|
||||
afor_df = pd.read_csv(infile_afor, sep = ',')
|
||||
snpinfo_df = pd.read_csv(infile_snpinfo, sep = ',')
|
||||
afor_df.columns = afor_df.columns.str.lower()
|
||||
|
||||
if afor_df['mutation'].shape[0] == afor_df['mutation'].nunique():
|
||||
print('No duplicate muts detected in afor_df')
|
||||
else:
|
||||
print('Dropping duplicate muts detected in afor_df')
|
||||
afor_df = afor_df.drop_duplicates(subset = 'mutation', keep = 'first')
|
||||
|
||||
|
||||
snpinfo_df_all = pd.read_csv(infile_snpinfo, sep = ',')
|
||||
snpinfo_df = snpinfo_df_all[['mutation', 'mutationinformation']]
|
||||
|
||||
|
||||
if snpinfo_df['mutation'].shape[0] == snpinfo_df['mutation'].nunique():
|
||||
print('No duplicate muts detected in snpinfo_df')
|
||||
else:
|
||||
dups = snpinfo_df['mutation'].duplicated().sum()
|
||||
print( dups, 'Duplicate muts detected in snpinfo_df'
|
||||
, '\nDim:', snpinfo_df.shape)
|
||||
print('Dropping duplicate muts')
|
||||
snpinfo_df = snpinfo_df.drop_duplicates(subset = 'mutation', keep = 'first')
|
||||
print('Dim:', snpinfo_df.shape)
|
||||
|
||||
|
||||
print('==================================='
|
||||
, '\nFifth merge: afor_df + snpinfo_df'
|
||||
, '\n===================================')
|
||||
|
||||
merging_cols_m5 = detect_common_cols(afor_df, snpinfo_df)
|
||||
|
||||
afor_snpinfo_dfs = pd.merge(afor_df, snpinfo_df, on = merging_cols_m5, how = 'left')
|
||||
#afor_df.shape
|
||||
#snpinfo_df.shape
|
||||
if len(afor_snpinfo_dfs) == afor_df.shape[0]:
|
||||
print('PASS: succesfully combined with left join')
|
||||
else:
|
||||
sys.exit('FAIL: unsuccessful merge')
|
||||
|
||||
#%%
|
||||
|
||||
afor_kin_df = pd.read_csv(infile_afor_kin, sep = ',')
|
||||
afor_kin_df.columns = afor_kin_df.columns.str.lower()
|
||||
|
||||
print('==================================='
|
||||
, '\nSixth merge: afor_snpinfo_dfs + afor_kin_df'
|
||||
, '\n===================================')
|
||||
|
||||
merging_cols = ['alt_allele',
|
||||
'chr_num_allele',
|
||||
'chromosome_number',
|
||||
'gene_id',
|
||||
'gene_number',
|
||||
'mut_info',
|
||||
'mut_region',
|
||||
'mut_type',
|
||||
'mutant_type',
|
||||
'mutationinformation',
|
||||
'position',
|
||||
'ref_allele',
|
||||
'wild_type']
|
||||
merging_cols_m6 = detect_common_cols(afor_snpinfo_dfs, afor_kin_df)
|
||||
|
||||
print('doing thing')
|
||||
print('Dim of df1:', afor_snpinfo_dfs.shape
|
||||
, '\nDim of df2:', afor_kin_df.shape
|
||||
, '\nno. of merging_cols:', len(merging_cols_m6))
|
||||
|
||||
ors_df = pd.merge(afor_snpinfo_dfs, afor_kin_df, on = merging_cols_m6, how = 'outer')
|
||||
|
||||
print('Dim of ors_df:', ors_df.shape)
|
||||
|
||||
#%%
|
||||
|
||||
print('==================================='
|
||||
, '\nSeventh merge: combined_df + ors_df'
|
||||
, '\n===================================')
|
||||
|
||||
merging_cols_m7 = detect_common_cols(combined_df, ors_df)
|
||||
|
||||
print('Dim of df1:', combined_df.shape
|
||||
, '\nDim of df2:', ors_df.shape
|
||||
, '\nno. of merging_cols:', len(merging_cols_m7))
|
||||
|
||||
print('checking mutations in the two dfs:'
|
||||
, '\nmuts in df1 but NOT in df2:'
|
||||
, combined_df['mutationinformation'].isin(ors_df['mutationinformation']).sum()
|
||||
, 'muts in df2 but NOT in df1:'
|
||||
, ors_df['mutationinformation'].isin(combined_df['mutationinformation']).sum())
|
||||
|
||||
#print('\nNo. of common muts:', np.intersect1d(combined_df['mutationinformation'], ors_df['mutationinformation']) )
|
||||
|
||||
#combined_df_all = pd.merge(combined_df, ors_df, on = merging_cols_m7, how = 'outer') # FIXME
|
||||
combined_df_all = pd.merge(combined_df, ors_df, on = merging_cols_m7, how = 'left')
|
||||
|
||||
outdf_expected_rows = len(combined_df)
|
||||
outdf_expected_cols = len(combined_df.columns) + len(ors_df.columns) - len(merging_cols_m7)
|
||||
|
||||
print('\nDim of combined_df_all:', combined_df_all.shape)
|
||||
|
||||
if combined_df_all.shape[1] == outdf_expected_cols:
|
||||
print('combined_df has expected no. of cols')
|
||||
if combined_df_all.shape[0] == outdf_expected_rows:
|
||||
print('combined_df has expected no. of rows')
|
||||
else:
|
||||
print('WARNING: nrows discrepancy noted'
|
||||
, '\nFIX IT')
|
||||
|
||||
|
||||
comb_afor_snpinfo = pd.merge(afor_df, snpinfo_df, on = 'mutation', how = 'inner')
|
||||
comb2 = pd.merge(comb_afor_snpinfo, afor_kin_df, on = merging_cols, how = i_join)
|
||||
comb3 = comb2.drop_duplicates(subset=merging_cols, keep = 'first')
|
||||
|
||||
common = np.intersect1d(comb_afor_snpinfo['mutationinformation'], afor_kin_df['mutationinformation'])
|
||||
|
||||
print('comb3 dim:', comb3.shape
|
||||
, '\ncomb2 dim:', comb2.shape
|
||||
, '\ndim of df1:', comb_afor_snpinfo.shape
|
||||
, '\ndim of df2:', afor_kin_df.shape
|
||||
, '\ncommon vals:', len(common))
|
||||
|
||||
print('expected:\n')
|
||||
|
||||
|
||||
|
||||
|
||||
bar = combine_stability_dfs(comb_afor_snpinfo, afor_kin_df, my_join = o_join)
|
||||
print('XXXXXX\n:', bar.shape)
|
||||
|
||||
#bar = np.intersect1d(comb_afor_snpinfo[merging_cols[0]], afor_kin_df[merging_cols[0]])
|
||||
#print('common values:',len(bar))
|
||||
#comb2 = combine_stability_dfs(comb_afor_snpinfo, afor_kin_df, my_join = o_join)
|
||||
print ('thing finished')
|
||||
#%% write csv
|
||||
|
||||
combined_df_all.to_csv(outfile_comb, index = False)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue