From 262bd79204a91199770db2fa9de2d7628c0cb1bb Mon Sep 17 00:00:00 2001 From: Tanushree Tunstall Date: Fri, 3 Jul 2020 19:22:46 +0100 Subject: [PATCH] stil fiddling iwth combining dfs --- scripts/combining.py | 194 +++++++++++++++++++------------- scripts/combining_mcsm_foldx.py | 10 +- 2 files changed, 118 insertions(+), 86 deletions(-) diff --git a/scripts/combining.py b/scripts/combining.py index 8bab131..a0c05c1 100755 --- a/scripts/combining.py +++ b/scripts/combining.py @@ -8,69 +8,74 @@ Created on Tue Aug 6 12:56:03 2019 # FIXME: change filename 2(mcsm normalised data) # to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline #======================================================================= -# Task: combine 2 dfs with aa position as linking column +# Task: combine 2 dfs on comm_valson cols by detecting them +# includes sainity checks -# Input: 2 dfs -# _complex_mcsm_norm.csv -# _foldx.csv - -# Output: .csv of all 2 dfs combined - -# useful link -# https://stackoverflow.com/questions/23668427/pandas-three-way-joining-multiple-dataframes-on-columns #======================================================================= #%% load packages import sys, os import pandas as pd import numpy as np +import re #from varname import nameof #%% end of variable assignment for input and output files #======================================================================= -#%% function/methd to combine 4 dfs +#%% function/methd to combine dfs -#def combine_stability_dfs(mcsm_df, foldx_df, out_combined_df): -def combine_stability_dfs(mcsm_df, foldx_df, my_join = 'outer'): +def detect_common_cols (df1, df2): """ - Combine 2 dfs - - @param mcsm_df: csv file (output from mcsm pipeline) - @type mcsm_df: string - - @param foldx_df: csv file (output from runFoldx.py) - @type foldx_df: string + Detect comm_valson cols - @param out_combined_df: csv file output - @type out_combined_df: string + @param df1: df + @type df1: pandas df + + @param df2: df + @type df2: pandas df + + @return: comm_valson cols + @type: list + """ + common_cols = np.intersect1d(df1.columns, df2.columns).tolist() + #print('Length of comm_cols:', len(comm_cols) + # , '\nmerging column/s:', comm_cols + # , '\ntype:', type(comm_cols) + # , '\ndtypes in merging columns:\n', df1[comm_cols].dtypes) + + return common_cols + + +def combine_stability_dfs(df1, df2, my_join = 'outer'): + """ + Combine 2 dfs by finding merging columns automatically + + @param df1: data frame + @type df1: pandas df + + @param df2: data frame + @type df2: pandas df + + @my_join: join type for merging + @type my_join: string - @return: none, writes combined df as csv + @return: combined_df + @type: pandas df """ - #======================== - # read input csv files to combine - #======================== - print('Reading input files:') - - left_df = pd.read_csv(mcsm_df, sep = ',') - left_df.columns = left_df.columns.str.lower() - - right_df = pd.read_csv(foldx_df, sep = ',') - right_df.columns = right_df.columns.str.lower() - - print('Dimension left df:', left_df.shape - , '\nDimesnion right_df:', right_df.shape -# , '\njoin type:', join_type - , '\n=========================================================') - - print('Finding common cols and merging cols:' + + print('Finding comm_valson cols and merging cols:' ,'\n=========================================================') - - common_cols = np.intersect1d(left_df.columns, right_df.columns).tolist() - print('Length of common cols:', len(common_cols) - , '\ncommon column/s:', common_cols, 'type:', type(common_cols)) + common_cols = np.intersect1d(df1.columns, df2.columns).tolist() + print('Length of comm_valson cols:', len(common_cols) + , '\nmerging column/s:', common_cols + , '\ntype:', type(common_cols) + , '\ndtypes in merging columns:\n', df1[common_cols].dtypes) print('selecting consistent dtypes for merging (object i.e string)') - merging_cols = left_df[common_cols].select_dtypes(include = [object]).columns.tolist() + #merging_cols = df1[comm_valson_cols].select_dtypes(include = [object]).columns.tolist() + #merging_cols = df1[comm_valson_cols].select_dtypes(include = ['int64']).columns.tolist() + merging_cols = common_cols.copy() + nmerging_cols = len(merging_cols) print(' length of merging cols:', nmerging_cols , '\nmerging cols:', merging_cols, 'type:', type(merging_cols) @@ -79,67 +84,94 @@ def combine_stability_dfs(mcsm_df, foldx_df, my_join = 'outer'): #======================== # merge 1 (combined_df) # concatenating 2dfs: - # mcsm_df, foldx_df + # df1, df2 #======================== # checking cross-over of mutations in the two dfs to merge - #ndiff1 = left_df.shape[0] - left_df['mutationinformation'].isin(right_df['mutationinformation']).sum() - ndiff_1 = left_df[merging_cols].squeeze().isin(right_df[merging_cols].squeeze()).sum() - print('ndiff_1:', ndiff_1) - - ndiff1 = left_df.shape[0] - ndiff_1 - #print('There are', ndiff1, 'unmatched mutations in left df') + ndiff_1 = df1[merging_cols].squeeze().isin(df2[merging_cols].squeeze()).sum() + ndiff1 = df1.shape[0] - ndiff_1 + print('There are', ndiff1, 'unmatched mutations in left df') - #missing_mutinfo = left_df[~left_df['mutationinformation'].isin(right_df['mutationinformation'])] + #missing_mutinfo = df1[~left_df['mutationinformation'].isin(df2['mutationinformation'])] #missing_mutinfo.to_csv('infoless_muts.csv') - - #ndiff2 = right_df.shape[0] - right_df['mutationinformation'].isin(left_df['mutationinformation']).sum() - ndiff_2 = right_df[merging_cols].squeeze().isin(left_df[merging_cols].squeeze()).sum() - print('ndiff_2:', ndiff_2) - - ndiff2 = right_df.shape[0] - ndiff_2 - #print('There are', ndiff2, 'unmatched mutations in right_df') - comm = np.intersect1d(left_df[merging_cols], right_df[merging_cols]) - comm_count = len(comm) - print('inner:', comm, '\nlength:', comm_count , '\ntype:', type(comm_count)) - + ndiff_2 = df2[merging_cols].squeeze().isin(df1[merging_cols].squeeze()).sum() + ndiff2 = df2.shape[0] - ndiff_2 + print('There are', ndiff2, 'unmatched mutations in right_df') + + #comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols]) + #comm_vals_count = len(comm_vals) + #print('length of comm_valson values:', comm_vals_count , '\ntype:', type(comm_vals_count)) + #======================== - # sanity checks for join type + # merging dfs & sanity checks #======================== fail = False print('combing with:', my_join) - combined_df = pd.merge(left_df, right_df, on = merging_cols, how = my_join) - combined_df1 = combined_df.drop_duplicates(subset = merging_cols, keep ='first') + comb_df = pd.merge(df1, df2, on = merging_cols, how = my_join) + combined_df = comb_df.drop_duplicates(subset = merging_cols, keep ='first') + + expected_cols = df1.shape[1] + df2.shape[1] - nmerging_cols - if my_join == 'inner': - #expected_rows = left_df.shape[0] - ndiff1 - expected_rows = comm_count - - if my_join == 'outer': - #expected_rows = right_df.shape[0] + ndiff1 - expected_rows = max(left_df.shape[0], right_df.shape[0]) if my_join == 'right': - expected_rows = right_df.shape[0] + df2_nd = df2.drop_duplicates(merging_cols, keep = 'first') + expected_rows = df2_nd.shape[0] if my_join == 'left': - expected_rows = left_df.shape[0] + expected_rows = df1.shape[0] - expected_cols = left_df.shape[1] + right_df.shape[1] - nmerging_cols - - if len(combined_df1) == expected_rows and len(combined_df1.columns) == expected_cols: + + #if my_join == 'inner': + # expected_rows = comm_vals_count + + #if my_join == 'outer': + # df1_nd = df1.drop_duplicates(merging_cols, keep = 'first') + # df2_nd = df2.drop_duplicates(merging_cols, keep = 'first') + # expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - comm_vals_count + + + if my_join == 'inner' or 'outer' and len(merging_cols)>1: + comm_vals = np.intersect1d(df1['mutationinformation'], df2['mutationinformation']) + print('length of comm_values for merge:', len(comm_vals)) + if my_join == 'inner': + expected_rows = len(comm_vals) + if my_join == 'outer': + df1_nd = df1.drop_duplicates(merging_cols, keep = 'first') + df2_nd = df2.drop_duplicates(merging_cols, keep = 'first') + expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - len(comm_vals) + else: + comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols]) + print('length of comm_values for merge:', len(comm_vals)) + if my_join == 'inner': + expected_rows = len(comm_vals) + if my_join == 'outer': + df1_nd = df1.drop_duplicates(merging_cols, keep = 'first') + df2_nd = df2.drop_duplicates(merging_cols, keep = 'first') + expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - len(comm_vals) + + if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols: print('PASS: successfully combined dfs with:', my_join, 'join') else: print('FAIL: combined_df\'s expected rows and cols not matched') fail = True print('\nExpected no. of rows:', expected_rows - , '\nGot:', len(combined_df1) + , '\nGot:', len(combined_df) , '\nExpected no. of cols:', expected_cols - , '\nGot:', len(combined_df1.columns)) + , '\nGot:', len(combined_df.columns)) if fail: sys.exit() - return combined_df1 + #if clean: + #foo = combined_df2.filter(regex = r'.*_x|_y', axis = 1) + #print(foo.columns) + #print('Detected duplicate cols with suffix: _x _y' + # , '\Dropping duplicate cols and cleaning') + + # drop position col containing suffix '_y' and then rename col without suffix + combined_df_clean = combined_df.drop(combined_df.filter(regex = r'.*_y').columns, axis = 1) + combined_df_clean.rename(columns=lambda x: re.sub('_x$','', x), inplace = True) + + return combined_df_clean #%% end of function #======================================================================= diff --git a/scripts/combining_mcsm_foldx.py b/scripts/combining_mcsm_foldx.py index 434ab92..c5b846c 100755 --- a/scripts/combining_mcsm_foldx.py +++ b/scripts/combining_mcsm_foldx.py @@ -52,7 +52,7 @@ gene = args.gene # dirs #====== datadir = homedir + '/' + 'git/Data' -indir = datadir + '/' + drug + '/' + 'output' +indir = datadir + '/' + drug + '/' + 'input' outdir = datadir + '/' + drug + '/' + 'output' #======= @@ -61,10 +61,10 @@ outdir = datadir + '/' + drug + '/' + 'output' in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv' in_filename_foldx = gene.lower() + '_foldx.csv' -infile_mcsm = indir + '/' + in_filename_mcsm -infile_foldx = indir + '/' + in_filename_foldx +infile_mcsm = outdir + '/' + in_filename_mcsm +infile_foldx = outdir + '/' + in_filename_foldx -print('\nInput path:', indir +print('\nInput path:', outdir , '\nInput filename1:', in_filename_mcsm , '\nInput filename2:', in_filename_foldx , '\n============================================================') @@ -109,4 +109,4 @@ def main(): if __name__ == '__main__': main() #======================================================================= -#%% end of script \ No newline at end of file +#%% end of script