resolving merge conflicts dur to shoddy data

scripts/combining_FIXME.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+'''
+Created on Tue Aug  6 12:56:03 2019
+
+@author: tanu
+'''
+# FIXME: change filename  2(mcsm normalised data)
+# to be consistent like (pnca_complex_mcsm_norm.csv) : changed manually, but ensure this is done in the mcsm pipeline
+#=======================================================================
+# Task: combine 2 dfs on comm_valson cols by detecting them
+# includes sainity checks
+
+#=======================================================================
+#%% load packages
+import sys, os
+import pandas as pd
+import numpy as np
+import re
+#from varname import nameof
+
+#%% end of variable assignment for input and output files
+#=======================================================================
+#%% function/methd to combine dfs
+
+def detect_common_cols (df1, df2):
+    """
+    Detect comm_valson cols
+    
+    @param df1: df
+    @type df1: pandas df
+
+    @param df2: df
+    @type df2: pandas df
+          
+    @return: comm_valson cols
+    @type: list
+    """
+    common_cols  = np.intersect1d(df1.columns, df2.columns).tolist()
+    print('Length of comm_cols:', len(common_cols)
+          , '\nMerging column/s:', common_cols
+          , '\n---------------------------------------------------------------'
+          , '\nType:', type(common_cols)
+          , '\n\ndtypes in merging columns:\n', df1[common_cols].dtypes
+          , '\n---------------------------------------------------------------')
+    
+    return common_cols
+    
+#%% Function to combine 2 dfs by detecting commom cols and performing 
+# sanity checks on the output df
+def combine_dfs_with_checks(df1, df2, my_join = 'outer'):
+    """
+    Combine 2 dfs by finding merging columns automatically
+
+    @param df1: data frame
+    @type df1: pandas df
+
+    @param df2: data frame
+    @type df2: pandas df
+    
+    @my_join: join type for merging
+    @type my_join: string
+        
+    @return:  combined_df
+    @type: pandas df
+    """
+        
+    print('Finding comm_cols and merging cols:'
+          ,'\n=========================================================')
+
+    common_cols  = np.intersect1d(df1.columns, df2.columns).tolist()
+    print('Length of comm_cols:', len(common_cols)
+          , '\nmerging column/s:', common_cols
+          , '\ntype:', type(common_cols))
+    
+    #print('\ndtypes in merging columns:\n', df1[common_cols].dtypes)
+    
+    print('selecting consistent dtypes for merging (object i.e string)')
+    #merging_cols = df1[comm_valson_cols].select_dtypes(include = [object]).columns.tolist()
+    #merging_cols = df1[comm_valson_cols].select_dtypes(include = ['int64']).columns.tolist()
+    merging_cols = common_cols.copy()
+
+    nmerging_cols = len(merging_cols)
+    print(' length of merging cols:', nmerging_cols
+      , '\nmerging cols:', merging_cols, 'type:', type(merging_cols)
+      , '\n=========================================================')
+    
+    #========================
+    # merge 1 (combined_df)
+    # concatenating 2dfs:
+    # df1, df2
+    #========================
+    # checking cross-over of mutations in the two dfs to merge
+    ndiff_1 = df1[merging_cols].squeeze().isin(df2[merging_cols].squeeze()).sum()
+    ndiff1 =  df1.shape[0] - ndiff_1
+    print('There are', ndiff1, 'unmatched mutations in left df')
+   
+    #missing_mutinfo = df1[~left_df['mutationinformation'].isin(df2['mutationinformation'])]
+    #missing_mutinfo.to_csv('infoless_muts.csv')
+
+    ndiff_2 = df2[merging_cols].squeeze().isin(df1[merging_cols].squeeze()).sum()
+    ndiff2 = df2.shape[0] - ndiff_2
+    print('There are', ndiff2, 'unmatched mutations in right_df')
+
+    #comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols])
+    #comm_vals_count = len(comm_vals)
+    #print('length of comm_valson values:', comm_vals_count , '\ntype:', type(comm_vals_count))
+
+    #========================
+    # merging dfs & sanity checks
+    #========================
+    fail = False 
+    print('combing with:', my_join)
+    comb_df = pd.merge(df1, df2, on = merging_cols, how = my_join)
+        
+    expected_cols = df1.shape[1] + df2.shape[1] - nmerging_cols
+    
+    
+    if my_join == 'right':
+        df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
+        expected_rows = df2_nd.shape[0]
+        
+    if my_join == 'left':
+        expected_rows = df1.shape[0]
+    
+    
+    #if my_join == 'inner':
+    #    expected_rows = comm_vals_count
+        
+    #if my_join == 'outer':
+    #    df1_nd = df1.drop_duplicates(merging_cols, keep = 'first')
+    #    df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
+    #    expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - comm_vals_count
+    
+        
+    if my_join == ('inner' or 'outer') and len(merging_cols) > 1:
+        #comm_vals = np.intersect1d(df1['mutationinformation'], df2['mutationinformation'])
+        print('length of merging_cols > 1, therefore omitting row checks')
+        combined_df = comb_df.copy()
+        expected_rows = len(combined_df)    
+        
+    else:
+        comm_vals = np.intersect1d(df1[merging_cols], df2[merging_cols])
+        print('length of merging_cols == 1, calculating expected rows in merged_df')
+        combined_df = comb_df.drop_duplicates(subset = merging_cols, keep ='first')
+        if my_join == 'inner':
+            expected_rows = len(comm_vals)
+        if my_join == 'outer':
+            df1_nd = df1.drop_duplicates(merging_cols, keep = 'first')
+            df2_nd = df2.drop_duplicates(merging_cols, keep = 'first')
+            expected_rows = df1_nd.shape[0] + df2_nd.shape[0] - len(comm_vals)  
+     
+    if len(combined_df) == expected_rows and len(combined_df.columns) == expected_cols:
+        print('PASS: successfully combined dfs with:', my_join, 'join')
+    else:
+        print('FAIL: combined_df\'s expected rows and cols not matched')
+        fail = True 
+    print('\nExpected no. of rows:', expected_rows
+          , '\nGot:', len(combined_df)
+          , '\nExpected no. of cols:', expected_cols
+          , '\nGot:', len(combined_df.columns))
+    if fail:
+        sys.exit()
+    
+    #if clean:
+    #foo = combined_df2.filter(regex = r'.*_x|_y', axis = 1)
+    #print(foo.columns)
+    #print('Detected duplicate cols with suffix: _x _y'
+    #      , '\Dropping duplicate cols and cleaning')
+    
+    # drop position col containing suffix '_y' and then rename col without suffix
+    combined_df_clean = combined_df.drop(combined_df.filter(regex = r'.*_y').columns, axis = 1)
+    combined_df_clean.rename(columns=lambda x: re.sub('_x$','', x), inplace = True)
+    
+    return combined_df_clean
+
+#%% end of function
+#=======================================================================
+