tanu/LSHTM_analysis
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

stil fiddling iwth combining dfs

Browse source
This commit is contained in:
Tanushree Tunstall 2020-07-03 19:22:46 +01:00
parent 90cbb49560
commit 262bd79204
2 changed files with 118 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

178
scripts/combining.py
View file

@ -8,69 +8,74 @@ Created on Tue Aug 6 12:56:03 2019
# FIXME: change filename 2(mcsm normalised data) # FIXME: change filename 2(mcsm normalised data)
# to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline # to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline
#======================================================================= #=======================================================================
# Task: combine 2 dfs with aa position as linking column # Task: combine 2 dfs on comm_valson cols by detecting them
# includes sainity checks
# Input: 2 dfs
# <gene.lower()>_complex_mcsm_norm.csv
# <gene.lower()>_foldx.csv
# Output: .csv of all 2 dfs combined
# useful link
# https://stackoverflow.com/questions/23668427/pandas-three-way-joining-multiple-dataframes-on-columns
#======================================================================= #=======================================================================
#%% load packages #%% load packages
import sys, os import sys, os
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import re
#from varname import nameof #from varname import nameof
#%% end of variable assignment for input and output files #%% end of variable assignment for input and output files
#======================================================================= #=======================================================================
#%% function/methd to combine 4 dfs #%% function/methd to combine dfs
#def combine_stability_dfs(mcsm_df, foldx_df, out_combined_df): def detect_common_cols (df1, df2):
def combine_stability_dfs(mcsm_df, foldx_df, my_join = 'outer'):
""" """
Combine 2 dfs Detect comm_valson cols
@param mcsm_df: csv file (output from mcsm pipeline) @param df1: df
@type mcsm_df: string @type df1: pandas df
@param foldx_df: csv file (output from runFoldx.py) @param df2: df
@type foldx_df: string @type df2: pandas df
@param out_combined_df: csv file output @return: comm_valson cols
@type out_combined_df: string @type: list
@return: none, writes combined df as csv
""" """
#======================== common_cols = np.intersect1d(df1.columns, df2.columns).tolist()
# read input csv files to combine #print('Length of comm_cols:', len(comm_cols)
#======================== # , '\nmerging column/s:', comm_cols
print('Reading input files:') # , '\ntype:', type(comm_cols)
# , '\ndtypes in merging columns:\n', df1[comm_cols].dtypes)
left_df = pd.read_csv(mcsm_df, sep = ',') return common_cols
left_df.columns = left_df.columns.str.lower()
right_df = pd.read_csv(foldx_df, sep = ',')
right_df.columns = right_df.columns.str.lower()
print('Dimension left df:', left_df.shape def combine_stability_dfs(df1, df2, my_join = 'outer'):
, '\nDimesnion right_df:', right_df.shape """
# , '\njoin type:', join_type Combine 2 dfs by finding merging columns automatically
@param df1: data frame
@type df1: pandas df
@param df2: data frame
@type df2: pandas df
@my_join: join type for merging
@type my_join: string
@return: combined_df
@type: pandas df
"""
print('Finding comm_valson cols and merging cols:'
,'\n=========================================================') ,'\n=========================================================')
print('Finding common cols and merging cols:' common_cols = np.intersect1d(df1.columns, df2.columns).tolist()
,'\n=========================================================') print('Length of comm_valson cols:', len(common_cols)
, '\nmerging column/s:', common_cols
, '\ntype:', type(common_cols)
common_cols = np.intersect1d(left_df.columns, right_df.columns).tolist() , '\ndtypes in merging columns:\n', df1[common_cols].dtypes)
print('Length of common cols:', len(common_cols)
, '\ncommon column/s:', common_cols, 'type:', type(common_cols))
print('selecting consistent dtypes for merging (object i.e string)') print('selecting consistent dtypes for merging (object i.e string)')
merging_cols = left_df[common_cols].select_dtypes(include = [object]).columns.tolist() #merging_cols = df1[comm_valson_cols].select_dtypes(include = [object]).columns.tolist()
#merging_cols = df1[comm_valson_cols].select_dtypes(include = ['int64']).columns.tolist()
merging_cols = common_cols.copy()
nmerging_cols = len(merging_cols) nmerging_cols = len(merging_cols)
print(' length of merging cols:', nmerging_cols print(' length of merging cols:', nmerging_cols
, '\nmerging cols:', merging_cols, 'type:', type(merging_cols) , '\nmerging cols:', merging_cols, 'type:', type(merging_cols)
@ -79,67 +84,94 @@ def combine_stability_dfs(mcsm_df, foldx_df, my_join = 'outer'):
#======================== #========================
# merge 1 (combined_df) # merge 1 (combined_df)
# concatenating 2dfs: # concatenating 2dfs:
# mcsm_df, foldx_df # df1, df2
#======================== #========================
# checking cross-over of mutations in the two dfs to merge # checking cross-over of mutations in the two dfs to merge
#ndiff1 = left_df.shape[0] - left_df['mutationinformation'].isin(right_df['mutationinformation']).sum() ndiff_1 = df1[merging_cols].squeeze().isin(df2[merging_cols].squeeze()).sum()
ndiff_1 = left_df[merging_cols].squeeze().isin(right_df[merging_cols].squeeze()).sum() ndiff1 = df1.shape[0] - ndiff_1
print('ndiff_1:', ndiff_1) print('There are', ndiff1, 'unmatched mutations in left df')
ndiff1 = left_df.shape[0] - ndiff_1 #missing_mutinfo = df1[~left_df['mutationinformation'].isin(df2['mutationinformation'])]
#print('There are', ndiff1, 'unmatched mutations in left df')
#missing_mutinfo = left_df[~left_df['mutationinformation'].isin(right_df['mutationinformation'])]
#missing_mutinfo.to_csv('infoless_muts.csv') #missing_mutinfo.to_csv('infoless_muts.csv')
#ndiff2 = right_df.shape[0] - right_df['mutationinformation'].isin(left_df['mutationinformation']).sum() ndiff_2 = df2[merging_cols].squeeze().isin(df1[merging_cols].squeeze()).sum()
ndiff_2 = right_df[merging_cols].squeeze().isin(left_df[merging_cols].squeeze()).sum() ndiff2 = df2.shape[0] - ndiff_2
print('ndiff_2:', ndiff_2) print('There are', ndiff2, 'unmatched mutations in right_df')
ndiff2 = right_df.shape[0] - ndiff_2 #comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols])
#print('There are', ndiff2, 'unmatched mutations in right_df') #comm_vals_count = len(comm_vals)
#print('length of comm_valson values:', comm_vals_count , '\ntype:', type(comm_vals_count))
comm = np.intersect1d(left_df[merging_cols], right_df[merging_cols])
comm_count = len(comm)
print('inner:', comm, '\nlength:', comm_count , '\ntype:', type(comm_count))
#======================== #========================
# sanity checks for join type # merging dfs & sanity checks
#======================== #========================
fail = False fail = False
print('combing with:', my_join) print('combing with:', my_join)
combined_df = pd.merge(left_df, right_df, on = merging_cols, how = my_join) comb_df = pd.merge(df1, df2, on = merging_cols, how = my_join)
combined_df1 = combined_df.drop_duplicates(subset = merging_cols, keep ='first') combined_df = comb_df.drop_duplicates(subset = merging_cols, keep ='first')
if my_join == 'inner': expected_cols = df1.shape[1] + df2.shape[1] - nmerging_cols
#expected_rows = left_df.shape[0] - ndiff1
expected_rows = comm_count
if my_join == 'outer':
#expected_rows = right_df.shape[0] + ndiff1
expected_rows = max(left_df.shape[0], right_df.shape[0])
if my_join == 'right': if my_join == 'right':
expected_rows = right_df.shape[0] df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
expected_rows = df2_nd.shape[0]
if my_join == 'left': if my_join == 'left':
expected_rows = left_df.shape[0] expected_rows = df1.shape[0]
expected_cols = left_df.shape[1] + right_df.shape[1] - nmerging_cols
if len(combined_df1) == expected_rows and len(combined_df1.columns) == expected_cols: #if my_join == 'inner':
# expected_rows = comm_vals_count
#if my_join == 'outer':
# df1_nd = df1.drop_duplicates(merging_cols, keep = 'first')
# df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
# expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - comm_vals_count
if my_join == 'inner' or 'outer' and len(merging_cols)>1:
comm_vals = np.intersect1d(df1['mutationinformation'], df2['mutationinformation'])
print('length of comm_values for merge:', len(comm_vals))
if my_join == 'inner':
expected_rows = len(comm_vals)
if my_join == 'outer':
df1_nd = df1.drop_duplicates(merging_cols, keep = 'first')
df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - len(comm_vals)
else:
comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols])
print('length of comm_values for merge:', len(comm_vals))
if my_join == 'inner':
expected_rows = len(comm_vals)
if my_join == 'outer':
df1_nd = df1.drop_duplicates(merging_cols, keep = 'first')
df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - len(comm_vals)
if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols:
print('PASS: successfully combined dfs with:', my_join, 'join') print('PASS: successfully combined dfs with:', my_join, 'join')
else: else:
print('FAIL: combined_df\'s expected rows and cols not matched') print('FAIL: combined_df\'s expected rows and cols not matched')
fail = True fail = True
print('\nExpected no. of rows:', expected_rows print('\nExpected no. of rows:', expected_rows
, '\nGot:', len(combined_df1) , '\nGot:', len(combined_df)
, '\nExpected no. of cols:', expected_cols , '\nExpected no. of cols:', expected_cols
, '\nGot:', len(combined_df1.columns)) , '\nGot:', len(combined_df.columns))
if fail: if fail:
sys.exit() sys.exit()
return combined_df1 #if clean:
#foo = combined_df2.filter(regex = r'.*_x|_y', axis = 1)
#print(foo.columns)
#print('Detected duplicate cols with suffix: _x _y'
# , '\Dropping duplicate cols and cleaning')
# drop position col containing suffix '_y' and then rename col without suffix
combined_df_clean = combined_df.drop(combined_df.filter(regex = r'.*_y').columns, axis = 1)
combined_df_clean.rename(columns=lambda x: re.sub('_x$','', x), inplace = True)
return combined_df_clean
#%% end of function #%% end of function
#======================================================================= #=======================================================================

8
scripts/combining_mcsm_foldx.py
View file

@ -52,7 +52,7 @@ gene = args.gene
# dirs # dirs
#====== #======
datadir = homedir + '/' + 'git/Data' datadir = homedir + '/' + 'git/Data'
indir = datadir + '/' + drug + '/' + 'output' indir = datadir + '/' + drug + '/' + 'input'
outdir = datadir + '/' + drug + '/' + 'output' outdir = datadir + '/' + drug + '/' + 'output'
#======= #=======
@ -61,10 +61,10 @@ outdir = datadir + '/' + drug + '/' + 'output'
in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv' in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv'
in_filename_foldx = gene.lower() + '_foldx.csv' in_filename_foldx = gene.lower() + '_foldx.csv'
infile_mcsm = indir + '/' + in_filename_mcsm infile_mcsm = outdir + '/' + in_filename_mcsm
infile_foldx = indir + '/' + in_filename_foldx infile_foldx = outdir + '/' + in_filename_foldx
print('\nInput path:', indir print('\nInput path:', outdir
, '\nInput filename1:', in_filename_mcsm , '\nInput filename1:', in_filename_mcsm
, '\nInput filename2:', in_filename_foldx , '\nInput filename2:', in_filename_foldx
, '\n============================================================') , '\n============================================================')