From 01ef04613a7785dfb117704dd9f1dc09255e2d5c Mon Sep 17 00:00:00 2001 From: Tanushree Tunstall Date: Tue, 7 Jul 2020 15:46:13 +0100 Subject: [PATCH] renamed files that combine dfs --- scripts/combine_afs_ors.py | 393 ------------------ .../{combine_dfs.py => combine_struct_dfs.py} | 0 scripts/{combining.py => combining_dfs.py} | 40 +- scripts/combining_mcsm_foldx.py | 112 ----- scripts/combining_test.py | 264 +++++++----- 5 files changed, 187 insertions(+), 622 deletions(-) delete mode 100755 scripts/combine_afs_ors.py rename scripts/{combine_dfs.py => combine_struct_dfs.py} (100%) rename scripts/{combining.py => combining_dfs.py} (83%) delete mode 100755 scripts/combining_mcsm_foldx.py diff --git a/scripts/combine_afs_ors.py b/scripts/combine_afs_ors.py deleted file mode 100755 index 02a5243..0000000 --- a/scripts/combine_afs_ors.py +++ /dev/null @@ -1,393 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -''' -Created on Tue Aug 6 12:56:03 2019 - -@author: tanu -''' -# FIXME: change filename 4 (mcsm normalised data) -# to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline -#======================================================================= -# Task: combine 2 dfs with aa position as linking column -# This is done in 2 steps: -# merge 1: - -# useful link -# https://stackoverflow.com/questions/23668427/pandas-three-way-joining-multiple-dataframes-on-columns -#======================================================================= -#%% load packages -import sys, os -import pandas as pd -import numpy as np -import argparse -#======================================================================= -#%% specify input and curr dir -homedir = os.path.expanduser('~') - -# set working dir -os.getcwd() -os.chdir(homedir + '/git/LSHTM_analysis/scripts') -os.getcwd() - -# local import -#from reference_dict import my_aa_dict # CHECK DIR STRUC THERE! -from reference_dict import low_3letter_dict -#======================================================================= -#%% command line args -#arg_parser = argparse.ArgumentParser() -#arg_parser.add_argument('-d', '--drug', help = 'drug name', default = 'pyrazinamide') -#arg_parser.add_argument('-g', '--gene', help = 'gene name', default = 'pncA') # case sensitive -#args = arg_parser.parse_args() -#======================================================================= -#%% variable assignment: input and output -drug = 'pyrazinamide' -gene = 'pncA' -gene_match = gene + '_p.' - -# cmd variables -#drug = args.drug -#gene = args.gene -#gene_match = gene + '_p.' - -#========== -# dir -#========== -datadir = homedir + '/' + 'git/Data' -indir = datadir + '/' + drug + '/' + 'input' -outdir = datadir + '/' + drug + '/' + 'output' - -#======= -# input -#======= -in_filename_snpinfo = 'ns' + gene.lower() + '_snp_info.csv' -in_filename_afor = gene.lower() + '_af_or.csv' -in_filename_afor_kin = gene.lower() + '_af_or_kinship.csv' - -infile0 = indir + '/' + in_filename_snpinfo -infile1 = outdir + '/' + in_filename_afor -infile2 = outdir + '/' + in_filename_afor_kin - - -print('Input file0:', infile0 - , '\nInput file1:', infile1 - , '\nInput file2:', infile2 - , '\n=============================================================') - -#======= -# output -#======= -out_filename = gene.lower() + '_metadata_afs_ors.csv' -outfile = outdir + '/' + out_filename -print('Output file:', outfile - , '\n=============================================================') - - -del(in_filename_afor, in_filename_afor_kin, datadir, indir, outdir) -#%% end of variable assignment for input and output files -#======================================================================= -#%% format mutations -# mut_format: gene.abc1cde | 1A>1B - -#======================== -# read input csv files to combine -#======================== -snpinfo_df = pd.read_csv(infile0, sep = ',') -#snpinfo_ncols = len(snpinfo_df.columns) -#snpinfo.shape[0] = len(snpinfo_df) -print('No. of rows in', infile0, ':', snpinfo_df.shape[0] - , '\nNo. of cols in', infile0, ':', snpinfo_df.shape[1]) - -afor_df = pd.read_csv(infile1, sep = ',') -#afor_ncols = len(afor_df.columns) -#afor.shape[0] = len(afor_df) -print('No. of rows in', infile1, ':', afor_df.shape[0] - , '\nNo. of cols in', infile1, ':', afor_df.shape[1]) - -afor_kin_df = pd.read_csv(infile2, sep = ',') -#afor_kin.shape[0] = len(afor_kin_df) -#afor_kin_ncols = len(afor_kin_df.columns) -print('No. of rows in', infile2, ':', afor_kin_df.shape[0] - , '\nNo. of cols in', infile2, ':', afor_kin_df.shape[1]) - -#%% Process afor_df -#1) pull all snp_info so you have ref_allele, etc -# i.e merge afor_df and snpinfo_df -# find merging column - -left_df = afor_df.copy() -right_df = snpinfo_df.copy() - -common_cols = np.intersect1d(left_df.columns, right_df.columns).tolist() -print('Length of common cols:', len(common_cols) - , '\ncommon column/s:', common_cols, 'type:', type(common_cols)) - -#https://stackoverflow.com/questions/44639772/python-pandas-column-dtype-object-causing-merge-to-fail-with-dtypewarning-colu -print('selecting consistent dtypes for merging (object i.e string)') -merging_cols = left_df[common_cols].select_dtypes(include = [object]).columns.tolist() -print(merging_cols) -nmerging_cols = len(merging_cols) -print(' length of merging cols:', nmerging_cols - , '\nmerging cols:', merging_cols, 'type:', type(merging_cols)) - -#https://stackoverflow.com/questions/22720739/pandas-left-outer-join-results-in-table-larger-than-left-table -# drop duplicates else the expected rows don't match -print('Checking for duplicates in common col:', common_cols - , '\nNo of duplicates:' - , len(right_df[right_df.duplicated(common_cols)]) - , '\noriginal length:', right_df.shape[0]) - -right_df = right_df[~right_df.duplicated(common_cols)] -print('\nrevised length:', right_df.shape[0]) - -# checking cross-over of mutations in the two dfs to merge -ndiff1 = left_df.shape[0] - left_df['mutation'].isin(right_df['mutation']).sum() -print('There are', ndiff1, 'mutations with OR, but no snp_info' - , '\nExtracting and writing out file') -missing_mutinfo = left_df[~left_df['mutation'].isin(right_df['mutation'])] -#missing_mutinfo.to_csv('infoless_muts.csv') - -ndiff2 = right_df.shape[0] - right_df['mutation'].isin(left_df['mutation']).sum() -print('There are', ndiff2, 'mutations that do not have OR, but have snp_info') - -# Define join type -#my_join = 'inner' -#my_join = 'outer' -#my_join = 'right' -my_join = 'left' - -print('combing with join:', my_join) -combined_df1 = pd.merge(left_df, right_df, on = merging_cols, how = my_join) -print('\nshape:', combined_df1.shape) - -# inner = 252 -left_df.shape[0] - ndiff1 - -# outer = 331 -right_df.shape[0] + ndiff1 - -# right = 290 -right_df.shape[0] - -# left = 293 -left_df.shape[0] - - -#%% -# see if you want an extra clause here! -# Define join type -#my_join = 'inner' -#my_join = 'outer' -#my_join = 'right' -my_join = 'left' - -fail = False -print('combing with:', my_join) -combined_df1 = pd.merge(left_df, right_df, on = merging_cols, how = my_join) - -if my_join == 'inner': - #expected_rows = left_df.shape[0] - ndiff1 - expected_rows = left_df.shape[0] - ndiff1 - -if my_join == 'outer': - #expected_rows = right_df.shape[0] + ndiff1 - expected_rows = right_df.shape[0] + ndiff1 - -if my_join == 'right': - #expected_rows = right_df.shape[0] - expected_rows = right_df.shape[0] - -if my_join == 'left': - #expected_rows = left_df.shape[0] - expected_rows = left_df.shape[0] - -expected_cols = left_df.shape[1] + right_df.shape[1] - nmerging_cols - -if len(combined_df1) == expected_rows and len(combined_df1.columns) == expected_cols: - print('PASS: successfully combined dfs with:', my_join, 'join') -else: - print('FAIL: combined_df\'s expected rows and cols not matched') - fail = True -print('\nExpected no. of rows:', expected_rows - , '\nGot:', len(combined_df1) - , '\nExpected no. of cols:', expected_cols - , '\nGot:', len(combined_df1.columns)) -if fail: - sys.exit() - -# delete variables -del(left_df, right_df, common_cols, merging_cols, nmerging_cols, my_join, ndiff1, ndiff2, missing_mutinfo - , expected_rows, expected_cols, fail) -del(afor_df, snpinfo_df) -#======================================================================= - -#%% Second merge: combined_df1 and afor_kin_df - -left_df = combined_df1.copy() -right_df = afor_kin_df.copy() - -common_cols = np.intersect1d(left_df.columns, right_df.columns).tolist() -print('Length of common cols:', len(common_cols) - , '\ncommon column/s:', common_cols, 'type:', type(common_cols)) - -#https://stackoverflow.com/questions/44639772/python-pandas-column-dtype-object-causing-merge-to-fail-with-dtypewarning-colu -print('selecting consistent dtypes for merging (object i.e string)') - -#FIXME - -#merging_cols = left_df[common_cols].select_dtypes(include = [object]).columns.tolist() -merging_cols = ['wild_type', 'mutant_type', 'mutationinformation'] -nmerging_cols_cols = len(merging_cols) - -print(merging_cols) -nmerging_cols = len(merging_cols) -print(' length of merging cols:', nmerging_cols - , '\nmerging cols:', merging_cols, 'type:', type(merging_cols)) - -ndiff1 = left_df.shape[0] - left_df['mutationinformation'].isin(right_df['mutationinformation']).sum() -print('There are', ndiff1, 'mutations with OR, but not in OR kinship' - , '\nExtracting and writing out file') -missing_mutinfo = left_df[~left_df['mutationinformation'].isin(right_df['mutationinformation'])] -#missing_mutinfo.to_csv('infoless_muts.csv') - -ndiff2 = right_df.shape[0] - right_df['mutationinformation'].isin(left_df['mutationinformation']).sum() -print('There are', ndiff2, 'mutations that do not have OR, but have OR kinship') - -my_join = 'outer' - -fail = False -print('combing with:', my_join) -combined_df2 = pd.merge(left_df, right_df, on = merging_cols, how = my_join) - -if my_join == 'inner': - #expected_rows = left_df.shape[0] - ndiff1 - expected_rows = left_df.shape[0] - ndiff1 - -if my_join == 'outer': - #expected_rows = right_df.shape[0] + ndiff1 - expected_rows = right_df.shape[0] + ndiff1 - -if my_join == 'right': - #expected_rows = right_df.shape[0] - expected_rows = right_df.shape[0] - -if my_join == 'left': - #expected_rows = left_df.shape[0] - expected_rows = left_df.shape[0] - -expected_cols = left_df.shape[1] + right_df.shape[1] - nmerging_cols - -if len(combined_df2) == expected_rows and len(combined_df2.columns) == expected_cols: - print('PASS: successfully combined dfs with:', my_join, 'join') -else: - print('FAIL: combined_df\'s expected rows and cols not matched') - fail = True -print('\nExpected no. of rows:', expected_rows - , '\nGot:', len(combined_df2) - , '\nExpected no. of cols:', expected_cols - , '\nGot:', len(combined_df2.columns)) -if fail: - sys.exit() -#%% check duplicate cols: ones containing suffix '_x' or '_y' -# should only be position -foo = combined_df2.filter(regex = r'.*_x|_y', axis = 1) -print(foo.columns) # should only be position - -# drop position col containing suffix '_y' and then rename col without suffix -combined_or_df = combined_df2.drop(combined_df2.filter(regex = r'.*_y').columns, axis = 1) -#combined_or_df['position_x'].head() - -# renaming columns -#combined_or_df.rename(columns = {'position_x': 'position'}, inplace = True) -#combined_or_df['position'].head() -#recheck -#foo = combined_or_df.filter(regex = r'.*_x|_y', axis = 1) -#print(foo.columns) # should only be empty - - -# remove '_x' from some cols - -import re -def clean_colnames(colname): - - if re.search('.*_x', colname): - pos = re.search('.*_x', colname).start() - return colname[:pos] - else: - return colname - -#https://stackoverflow.com/questions/26500156/renaming-column-in-dataframe-for-pandas-using-regular-expression -combined_or_df.columns -combined_or_df.rename(columns=lambda x: re.sub('_x$','',x), inplace = True) -combined_or_df.columns - -#FIXME: this should be 0 when you run the 35k dataset -combined_or_df['chromosome_number'].isna().sum() - -#%% rearraging columns -print('Dim of df prefromatting:', combined_or_df.shape) - -print(combined_or_df.columns, '\nshape:', combined_or_df.shape) - -# removing unnecessary column -combined_or_df = combined_or_df.drop(['symbol'], axis = 1) -print(combined_or_df.columns, '\nshape:', combined_or_df.shape) -#%% reorder columns -#https://stackoverflow.com/questions/13148429/how-to-change-the-order-of-dataframe-columns -# setting column's order -output_df = combined_or_df[['mutation', - 'mutationinformation', - 'wild_type', - 'position', - 'mutant_type', - 'chr_num_allele', - 'ref_allele', - 'alt_allele', - 'mut_info', - 'mut_type', - 'gene_id', - 'gene_number', - 'mut_region', - 'reference_allele', - 'alternate_allele', - 'chromosome_number', - 'af', - 'af_kin', - 'or_kin', - 'or_logistic', - 'or_mychisq', - 'est_chisq', - 'or_fisher', - 'ci_low_logistic', - 'ci_hi_logistic', - 'ci_low_fisher', - 'ci_hi_fisher', - 'pwald_kin', - 'pval_logistic', - 'pval_fisher', - 'pval_chisq', - 'beta_logistic', - 'beta_kin', - 'se_logistic', - 'se_kin', - 'zval_logistic', - 'logl_H1_kin', - 'l_remle_kin', - 'wt_3let', - 'mt_3let', - 'n_diff', - 'tot_diff', - 'n_miss']] - -# sanity check after rearranging -if combined_or_df.shape == output_df.shape and set(combined_or_df.columns) == set(output_df.columns): - print('PASS: Successfully formatted df with rearranged columns') -else: - sys.exit('FAIL: something went wrong when rearranging columns!') - -#%% write file -print('\n=====================================================================' - , '\nWriting output file:\n', outfile - , '\nNo.of rows:', len(output_df) - , '\nNo. of cols:', len(output_df.columns)) -output_df.to_csv(outfile, index = False) - diff --git a/scripts/combine_dfs.py b/scripts/combine_struct_dfs.py similarity index 100% rename from scripts/combine_dfs.py rename to scripts/combine_struct_dfs.py diff --git a/scripts/combining.py b/scripts/combining_dfs.py similarity index 83% rename from scripts/combining.py rename to scripts/combining_dfs.py index a0c05c1..ce19fb0 100755 --- a/scripts/combining.py +++ b/scripts/combining_dfs.py @@ -37,15 +37,15 @@ def detect_common_cols (df1, df2): @type: list """ common_cols = np.intersect1d(df1.columns, df2.columns).tolist() - #print('Length of comm_cols:', len(comm_cols) - # , '\nmerging column/s:', comm_cols - # , '\ntype:', type(comm_cols) - # , '\ndtypes in merging columns:\n', df1[comm_cols].dtypes) + print('Length of comm_cols:', len(common_cols) + , '\nmerging column/s:', common_cols + , '\ntype:', type(common_cols) + , '\ndtypes in merging columns:\n', df1[common_cols].dtypes) return common_cols -def combine_stability_dfs(df1, df2, my_join = 'outer'): +def combine_dfs_with_checks(df1, df2, my_join = 'outer'): """ Combine 2 dfs by finding merging columns automatically @@ -62,14 +62,15 @@ def combine_stability_dfs(df1, df2, my_join = 'outer'): @type: pandas df """ - print('Finding comm_valson cols and merging cols:' + print('Finding comm_cols and merging cols:' ,'\n=========================================================') common_cols = np.intersect1d(df1.columns, df2.columns).tolist() - print('Length of comm_valson cols:', len(common_cols) + print('Length of comm_cols:', len(common_cols) , '\nmerging column/s:', common_cols - , '\ntype:', type(common_cols) - , '\ndtypes in merging columns:\n', df1[common_cols].dtypes) + , '\ntype:', type(common_cols)) + + #print('\ndtypes in merging columns:\n', df1[common_cols].dtypes) print('selecting consistent dtypes for merging (object i.e string)') #merging_cols = df1[comm_valson_cols].select_dtypes(include = [object]).columns.tolist() @@ -108,8 +109,7 @@ def combine_stability_dfs(df1, df2, my_join = 'outer'): fail = False print('combing with:', my_join) comb_df = pd.merge(df1, df2, on = merging_cols, how = my_join) - combined_df = comb_df.drop_duplicates(subset = merging_cols, keep ='first') - + expected_cols = df1.shape[1] + df2.shape[1] - nmerging_cols @@ -130,18 +130,16 @@ def combine_stability_dfs(df1, df2, my_join = 'outer'): # expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - comm_vals_count - if my_join == 'inner' or 'outer' and len(merging_cols)>1: - comm_vals = np.intersect1d(df1['mutationinformation'], df2['mutationinformation']) - print('length of comm_values for merge:', len(comm_vals)) - if my_join == 'inner': - expected_rows = len(comm_vals) - if my_join == 'outer': - df1_nd = df1.drop_duplicates(merging_cols, keep = 'first') - df2_nd = df2.drop_duplicates(merging_cols, keep = 'first') - expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - len(comm_vals) + if my_join == ('inner' or 'outer') and len(merging_cols) > 1: + #comm_vals = np.intersect1d(df1['mutationinformation'], df2['mutationinformation']) + print('length of merging_cols > 1, therefore omitting row checks') + combined_df = comb_df.copy() + expected_rows = len(combined_df) + else: comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols]) - print('length of comm_values for merge:', len(comm_vals)) + print('length of merging_cols == 1, calculating expected rows in merged_df') + combined_df = comb_df.drop_duplicates(subset = merging_cols, keep ='first') if my_join == 'inner': expected_rows = len(comm_vals) if my_join == 'outer': diff --git a/scripts/combining_mcsm_foldx.py b/scripts/combining_mcsm_foldx.py deleted file mode 100755 index c5b846c..0000000 --- a/scripts/combining_mcsm_foldx.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -''' -Created on Tue Aug 6 12:56:03 2019 - -@author: tanu -''' -# FIXME: change filename 2(mcsm normalised data) -# to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline -#======================================================================= -# Task: combine 2 dfs with aa position as linking column - -# Input: 2 dfs -# _complex_mcsm_norm.csv -# _foldx.csv - -# Output: .csv of all 2 dfs combined - -# useful link -# https://stackoverflow.com/questions/23668427/pandas-three-way-joining-multiple-dataframes-on-columns -#======================================================================= -#%% load packages -import sys, os -import pandas as pd -import numpy as np -#from varname import nameof -import argparse -from combining import combine_stability_dfs -#======================================================================= -#%% specify input and curr dir -homedir = os.path.expanduser('~') - -# set working dir -os.getcwd() -os.chdir(homedir + '/git/LSHTM_analysis/scripts') -os.getcwd() -#======================================================================= -#%% command line args -arg_parser = argparse.ArgumentParser() -arg_parser.add_argument('-d', '--drug', help='drug name', default = 'pyrazinamide') -arg_parser.add_argument('-g', '--gene', help='gene name', default = 'pncA') # case sensitive -args = arg_parser.parse_args() -#======================================================================= -#%% variable assignment: input and output -#drug = 'pyrazinamide' -#gene = 'pncA' -#gene_match = gene + '_p.' - -drug = args.drug -gene = args.gene -#====== -# dirs -#====== -datadir = homedir + '/' + 'git/Data' -indir = datadir + '/' + drug + '/' + 'input' -outdir = datadir + '/' + drug + '/' + 'output' - -#======= -# input -#======= -in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv' -in_filename_foldx = gene.lower() + '_foldx.csv' - -infile_mcsm = outdir + '/' + in_filename_mcsm -infile_foldx = outdir + '/' + in_filename_foldx - -print('\nInput path:', outdir - , '\nInput filename1:', in_filename_mcsm - , '\nInput filename2:', in_filename_foldx - , '\n============================================================') - -#======= -# output -#======= -out_filename_comb = gene.lower() + '_mcsm_foldx.csv' -outfile_comb = outdir + '/' + out_filename_comb -print('Output filename:', outfile_comb - , '\n============================================================') - -my_join_type = 'outer' -#my_join_type = 'left' -#my_join_type = 'right' -#my_join_type = 'inner' - -# end of variable assignment for input and output files -#%% call function -#======================================================================= -#combine_stability_dfs(mcsm_df, foldx_df, outfile) -#======================================================================= -def main(): - - combined_df = combine_stability_dfs(infile_mcsm, infile_foldx, my_join = my_join_type) - print('Combining 2 dfs...' - , '\nArguments to function combine_stability_dfs:' - , '\ndf1:', in_filename_mcsm - , '\ndf2:', in_filename_foldx - , '\njoin_type:', my_join_type - , '\ncombined df sneak peak:\n' - , combined_df.head()) - - print('Writing output...') - - combined_df.to_csv(outfile_comb, index = False) - - print('Finished writing output file' - , '\nOutput file:', outfile_comb - , '\nDimensions:', combined_df.shape) - -if __name__ == '__main__': - main() -#======================================================================= -#%% end of script diff --git a/scripts/combining_test.py b/scripts/combining_test.py index 6cede69..19b9c51 100755 --- a/scripts/combining_test.py +++ b/scripts/combining_test.py @@ -25,8 +25,7 @@ import pandas as pd import numpy as np #from varname import nameof import argparse -from combining import combine_stability_dfs -from combining import detect_common_cols + #======================================================================= #%% specify input and curr dir homedir = os.path.expanduser('~') @@ -35,6 +34,10 @@ homedir = os.path.expanduser('~') os.getcwd() os.chdir(homedir + '/git/LSHTM_analysis/scripts') os.getcwd() + +# local imports +from combining_dfs import combine_dfs_with_checks +from combining_dfs import detect_common_cols #======================================================================= #%% command line args #arg_parser = argparse.ArgumentParser() @@ -60,33 +63,33 @@ outdir = datadir + '/' + drug + '/' + 'output' # input #======= #in_filename_linking = gene.lower() + '_linking_df.csv' -#in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv' -#in_filename_foldx = gene.lower() + '_foldx.csv' +in_filename_mcsm = gene.lower() + '_complex_mcsm_norm.csv' +in_filename_foldx = gene.lower() + '_foldx.csv' in_filename_dssp = gene.lower() + '_dssp.csv' in_filename_kd = gene.lower() + '_kd.csv' -#in_filename_rd = gene.lower() + '_rd.csv' +in_filename_rd = gene.lower() + '_rd.csv' in_filename_snpinfo = 'ns' + gene.lower() + '_snp_info.csv' in_filename_afor = gene.lower() + '_af_or.csv' in_filename_afor_kin = gene.lower() + '_af_or_kinship.csv' #infile_linking = outdir + '/' + in_filename_linking -#infile_mcsm = outdir + '/' + in_filename_mcsm -#infile_foldx = outdir + '/' + in_filename_foldx +infile_mcsm = outdir + '/' + in_filename_mcsm +infile_foldx = outdir + '/' + in_filename_foldx infile_dssp = outdir + '/' + in_filename_dssp infile_kd = outdir + '/' + in_filename_kd -#infile_rd = outdir + '/' + in_filename_rd +infile_rd = outdir + '/' + in_filename_rd infile_snpinfo = indir + '/' + in_filename_snpinfo infile_afor = outdir + '/' + in_filename_afor infile_afor_kin = outdir + '/' + in_filename_afor_kin print('\nInput path:', outdir -# , '\nInput filename1:', infile_mcsm -# , '\nInput filename2:', infile_foldx - , '\nInput filename2:', infile_dssp - , '\nInput filename2:', infile_kd -# , '\nInput filename2:', infile_rd + , '\nInput filename mcsm:', infile_mcsm + , '\nInput filename foldx:', infile_foldx + , '\nInput filename dssp:', infile_dssp + , '\nInput filename kd:', infile_kd + , '\nInput filename rd', infile_rd , '\nInput filename snp info:', infile_snpinfo , '\nInput filename af or:', infile_afor , '\nInput filename afor kinship:', infile_afor_kin @@ -95,10 +98,10 @@ print('\nInput path:', outdir #======= # output #======= -#out_filename_comb = gene.lower() + '_struct_params_TEST.csv' -#outfile_comb = outdir + '/' + out_filename_comb -#print('Output filename:', outfile_comb -# , '\n============================================================') +out_filename_comb = gene.lower() + '_all_params.csv' +outfile_comb = outdir + '/' + out_filename_comb +print('Output filename:', outfile_comb + , '\n============================================================') o_join = 'outer' l_join = 'left' @@ -111,6 +114,8 @@ i_join = 'inner' #======================================================================= # call function to detect common cols +# FIXME: do the OR combining in the end to iron out any problems +# Couldn't run the function combin #======================================================================= def main(): @@ -133,99 +138,166 @@ def main(): # , '\nmerging column/s:', merging_cols, 'type:', type(merging_cols) # , '\ndtypes in merging columns:', dssp_df[merging_cols].dtypes) - #combined_df1 = combine_stability_dfs(dssp_df, kd_df, my_join = o_join) + #combined_df1 = combine_dfs_with_checks(dssp_df, kd_df, my_join = o_join) #print('Dimensions of combined df:', combined_df1.shape # , '\nsneak peak:', combined_df1.head() # , '\ndtypes in cols:\n', combined_df1.dtypes) -#============================================================================= - afor_df = pd.read_csv(infile_afor, sep = ',') - afor_df.columns = afor_df.columns.str.lower() - - snpinfo_df = pd.read_csv(infile_snpinfo, sep = ',') - snpinfo_df.columns = snpinfo_df.columns.str.lower() - -# print('Dimension df1:', afor_df.shape -# , '\nDimension df2:', snpinfo_df.shape -# , '\njoin type:', l_join -# , '\n=========================================================') - - - # detect common cols - merging_cols = detect_common_cols(afor_df, snpinfo_df) - #print('Length of common cols:', len(merging_cols) - # , '\nmerging column/s:', merging_cols, 'type:', type(merging_cols) - # , '\ndtypes in merging columns:', snpinfo_df[merging_cols].dtypes) - - comb_afor_snpinfo = combine_stability_dfs(afor_df, snpinfo_df, my_join = l_join) - #print('Dimensions of combined df:', comb_afor_snpinfo.shape - # , '\nsneak peak:', comb_afor_snpinfo.head() - # , '\ndtypes in cols:\n', comb_afor_snpinfo.dtypes) - -#============================================================================= - afor_kin_df = pd.read_csv(infile_afor_kin, sep = ',') - afor_kin_df.columns = afor_kin_df.columns.str.lower() - - # detect common cols - merging_cols = detect_common_cols(comb_afor_snpinfo, afor_kin_df) - -# comb2 = combine_stability_dfs(comb_afor_snpinfo, afor_kin_df, my_join = o_join) - #print('Dimensions of combined df:', comb2.shape - # , '\nsneak peak:', comb2.head() - # , '\ndtypes in cols:\n', comb2.dtype) - - -if __name__ == '__main__': - main() +#if __name__ == '__main__': +# main() #======================================================================= #%% end of script -#hardocoded test +#hardcoded test + +mcsm_df = pd.read_csv(infile_mcsm, sep = ',') +mcsm_df.columns = mcsm_df.columns.str.lower() +foldx_df = pd.read_csv(infile_foldx , sep = ',') + +print('===================================' + , '\nFirst merge: mcsm + foldx' + , '\n===================================') +#mcsm_foldx_dfs = combine_dfs_with_checks(mcsm_df, foldx_df, my_join = o_join) +merging_cols_m1 = detect_common_cols(mcsm_df, foldx_df) + +mcsm_foldx_dfs = pd.merge(mcsm_df, foldx_df, on = merging_cols_m1, how = 'outer') +ncols_m1 = len(mcsm_foldx_dfs.columns) + +print('===================================' + , '\nSecond merge: dssp + kd' + , '\n===================================') dssp_df = pd.read_csv(infile_dssp, sep = ',') kd_df = pd.read_csv(infile_kd, sep = ',') +rd_df = pd.read_csv(infile_rd, sep = ',') +#dssp_kd_dfs = combine_dfs_with_checks(dssp_df, kd_df, my_join = o_join) +merging_cols_m2 = detect_common_cols(dssp_df, kd_df) + +dssp_kd_dfs = pd.merge(dssp_df, kd_df, on = merging_cols_m2, how = 'outer') + +print('===================================' + , '\nThird merge: dssp_kd_dfs + rd_df' + , '\n===================================') +#dssp_kd_rd_dfs = combine_dfs_with_checks(dssp_kd_dfs, rd_df, my_join = o_join) +merging_cols_m3 = detect_common_cols(dssp_df, kd_df) +dssp_kd_rd_dfs = pd.merge(dssp_kd_dfs, rd_df, on = merging_cols_m3, how = 'outer') + +ncols_m3 = len(dssp_kd_rd_dfs.columns) + +print('===================================' + , '\nFourth merge: First merge + Third merge' + , '\n===================================') +#combined_dfs = combine_dfs_with_checks(mcsm_foldx_dfs, dssp_kd_rd_dfs, my_join = i_join)# gives wrong! +merging_cols_m4 = detect_common_cols(mcsm_foldx_dfs, dssp_kd_rd_dfs) +combined_df_expected_cols = ncols_m1 + ncols_m3 - len(merging_cols_m4) + +combined_df = pd.merge(mcsm_foldx_dfs, dssp_kd_rd_dfs, on = merging_cols_m4, how = 'inner') + + +if len(combined_df) == len(mcsm_df) and len(combined_df.columns) == combined_df_expected_cols: + print('PASS: successfully combined 5 dfs' + , '\nnrows combined_df:', len(combined_df) + , '\ncols combined_df:', len(combined_df.columns)) +else: + sys.exit('FAIL: check individual df merges') + +#%% OR combining afor_df = pd.read_csv(infile_afor, sep = ',') -snpinfo_df = pd.read_csv(infile_snpinfo, sep = ',') +afor_df.columns = afor_df.columns.str.lower() + +if afor_df['mutation'].shape[0] == afor_df['mutation'].nunique(): + print('No duplicate muts detected in afor_df') +else: + print('Dropping duplicate muts detected in afor_df') + afor_df = afor_df.drop_duplicates(subset = 'mutation', keep = 'first') + + +snpinfo_df_all = pd.read_csv(infile_snpinfo, sep = ',') +snpinfo_df = snpinfo_df_all[['mutation', 'mutationinformation']] + + +if snpinfo_df['mutation'].shape[0] == snpinfo_df['mutation'].nunique(): + print('No duplicate muts detected in snpinfo_df') +else: + dups = snpinfo_df['mutation'].duplicated().sum() + print( dups, 'Duplicate muts detected in snpinfo_df' + , '\nDim:', snpinfo_df.shape) + print('Dropping duplicate muts') + snpinfo_df = snpinfo_df.drop_duplicates(subset = 'mutation', keep = 'first') + print('Dim:', snpinfo_df.shape) + + +print('===================================' + , '\nFifth merge: afor_df + snpinfo_df' + , '\n===================================') + +merging_cols_m5 = detect_common_cols(afor_df, snpinfo_df) + +afor_snpinfo_dfs = pd.merge(afor_df, snpinfo_df, on = merging_cols_m5, how = 'left') +#afor_df.shape +#snpinfo_df.shape +if len(afor_snpinfo_dfs) == afor_df.shape[0]: + print('PASS: succesfully combined with left join') +else: + sys.exit('FAIL: unsuccessful merge') + +#%% + afor_kin_df = pd.read_csv(infile_afor_kin, sep = ',') +afor_kin_df.columns = afor_kin_df.columns.str.lower() + +print('===================================' + , '\nSixth merge: afor_snpinfo_dfs + afor_kin_df' + , '\n===================================') -merging_cols = ['alt_allele', - 'chr_num_allele', - 'chromosome_number', - 'gene_id', - 'gene_number', - 'mut_info', - 'mut_region', - 'mut_type', - 'mutant_type', - 'mutationinformation', - 'position', - 'ref_allele', - 'wild_type'] +merging_cols_m6 = detect_common_cols(afor_snpinfo_dfs, afor_kin_df) -print('doing thing') +print('Dim of df1:', afor_snpinfo_dfs.shape + , '\nDim of df2:', afor_kin_df.shape + , '\nno. of merging_cols:', len(merging_cols_m6)) + +ors_df = pd.merge(afor_snpinfo_dfs, afor_kin_df, on = merging_cols_m6, how = 'outer') + +print('Dim of ors_df:', ors_df.shape) + +#%% + +print('===================================' + , '\nSeventh merge: combined_df + ors_df' + , '\n===================================') + +merging_cols_m7 = detect_common_cols(combined_df, ors_df) + +print('Dim of df1:', combined_df.shape + , '\nDim of df2:', ors_df.shape + , '\nno. of merging_cols:', len(merging_cols_m7)) + +print('checking mutations in the two dfs:' + , '\nmuts in df1 but NOT in df2:' + , combined_df['mutationinformation'].isin(ors_df['mutationinformation']).sum() + , 'muts in df2 but NOT in df1:' + , ors_df['mutationinformation'].isin(combined_df['mutationinformation']).sum()) + +#print('\nNo. of common muts:', np.intersect1d(combined_df['mutationinformation'], ors_df['mutationinformation']) ) + +#combined_df_all = pd.merge(combined_df, ors_df, on = merging_cols_m7, how = 'outer') # FIXME +combined_df_all = pd.merge(combined_df, ors_df, on = merging_cols_m7, how = 'left') + +outdf_expected_rows = len(combined_df) +outdf_expected_cols = len(combined_df.columns) + len(ors_df.columns) - len(merging_cols_m7) + +print('\nDim of combined_df_all:', combined_df_all.shape) + +if combined_df_all.shape[1] == outdf_expected_cols: + print('combined_df has expected no. of cols') +if combined_df_all.shape[0] == outdf_expected_rows: + print('combined_df has expected no. of rows') +else: + print('WARNING: nrows discrepancy noted' + , '\nFIX IT') -comb_afor_snpinfo = pd.merge(afor_df, snpinfo_df, on = 'mutation', how = 'inner') -comb2 = pd.merge(comb_afor_snpinfo, afor_kin_df, on = merging_cols, how = i_join) -comb3 = comb2.drop_duplicates(subset=merging_cols, keep = 'first') - -common = np.intersect1d(comb_afor_snpinfo['mutationinformation'], afor_kin_df['mutationinformation']) - -print('comb3 dim:', comb3.shape - , '\ncomb2 dim:', comb2.shape - , '\ndim of df1:', comb_afor_snpinfo.shape - , '\ndim of df2:', afor_kin_df.shape - , '\ncommon vals:', len(common)) - -print('expected:\n') - - - - -bar = combine_stability_dfs(comb_afor_snpinfo, afor_kin_df, my_join = o_join) -print('XXXXXX\n:', bar.shape) - -#bar = np.intersect1d(comb_afor_snpinfo[merging_cols[0]], afor_kin_df[merging_cols[0]]) -#print('common values:',len(bar)) -#comb2 = combine_stability_dfs(comb_afor_snpinfo, afor_kin_df, my_join = o_join) print ('thing finished') +#%% write csv + +combined_df_all.to_csv(outfile_comb, index = False)