209 lines
9 KiB
Python
209 lines
9 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
Created on Thu Mar 24 15:01:59 2022
|
|
|
|
@author: tanu
|
|
"""
|
|
import sys, os
|
|
import pandas as pd
|
|
import numpy as np
|
|
from statistics import mean, median, mode
|
|
from statistics import multimode
|
|
from collections import Counter
|
|
from tidy_split import tidy_split
|
|
|
|
#import math
|
|
|
|
# https://stackoverflow.com/questions/43321455/pandas-count-null-values-in-a-groupby-function
|
|
# https://stackoverflow.com/questions/33457191/python-pandas-gene_LF2frame-fill-nans-with-a-conditional-mean
|
|
# round up
|
|
# int(math.ceil(mean(foo)))
|
|
# https://stackoverflow.com/questions/33457191/python-pandas-gene_LF2frame-fill-nans-with-a-conditional-mean
|
|
# https://stackoverflow.com/questions/37189878/pandas-add-column-to-groupby-gene_LF2frame
|
|
# https://stackoverflow.com/questions/43847520/how-to-get-the-distinct-count-of-values-in-a-python-pandas-gene_LF2frame
|
|
#%% Read gene_LF2 and formatting
|
|
drug = "pyrazinamide"
|
|
|
|
gene_LF2 = pd.read_csv("/home/tanu/git/ML_AI_training/test_gene_LF2/sample_gene_LF2.csv")
|
|
gene_LF2.columns
|
|
|
|
gene_LF2.head()
|
|
#%% Quick checks: Lineage and sample count for each mutation
|
|
gene_LF2['id'].nunique()
|
|
gene_LF2['mutationinformation'].nunique()
|
|
total_id_ucount = gene_LF2['id'].nunique()
|
|
total_id_ucount
|
|
|
|
gene_LF2.groupby('mutationinformation')['lineage'].size()
|
|
gene_LF2.groupby('mutationinformation')['lineage_corrupt'].size()
|
|
gene_LF2.groupby('mutationinformation')['id'].size()
|
|
gene_LF2.groupby('mutationinformation')['lineage'].value_counts()
|
|
gene_LF2.groupby('mutationinformation')['lineage'].nunique()
|
|
#%% id count: add all id ids and count of unique ids per mutation
|
|
gene_LF2['id_list'] = gene_LF2['mutationinformation'].map(gene_LF2.groupby('mutationinformation')['id'].apply(list))
|
|
gene_LF2['id_ucount'] = gene_LF2['mutationinformation'].map(gene_LF2.groupby('mutationinformation')['id'].nunique())
|
|
gene_LF2[['mutationinformation', 'id', 'id_list', 'id_ucount']]
|
|
#%% Lineages: add all lineages and count of unique lineages per mutation
|
|
# Lineages good: lineage column has only a single lineage for each mutationinformation
|
|
gene_LF2['lineage']
|
|
gene_LF2['lineage_list'] = gene_LF2['mutationinformation'].map(gene_LF2.groupby('mutationinformation')['lineage'].apply(list))
|
|
gene_LF2['lineage_ucount'] = gene_LF2['mutationinformation'].map(gene_LF2.groupby('mutationinformation')['lineage'].nunique())
|
|
gene_LF2[['mutationinformation', 'lineage', 'lineage_list', 'lineage_ucount']]
|
|
|
|
# Lineage corrupt: lineage column has only multiple lineages for each mutationinformation separated by ';'
|
|
gene_LF2['lineage_corrupt']
|
|
# split using tidy_split()
|
|
gene_LF2_split = tidy_split(gene_LF2, 'lineage_corrupt', sep = ';')
|
|
# remove leading white space else these are counted as distinct mutations as well
|
|
#gene_LF2_split['lineage_corrupt'] = gene_LF2_split['lineage_corrupt'].str.lstrip()
|
|
gene_LF2_split['lineage_corrupt'] = gene_LF2_split['lineage_corrupt'].str.strip()
|
|
gene_LF2_split.head()
|
|
|
|
gene_LF2_split['lineage_corrupt_list'] = gene_LF2_split['mutationinformation'].map(gene_LF2_split.groupby('mutationinformation')['lineage_corrupt'].apply(list))
|
|
gene_LF2_split['lineage_corrupt_ucount'] = gene_LF2_split['mutationinformation'].map(gene_LF2_split.groupby('mutationinformation')['lineage_corrupt'].nunique())
|
|
|
|
gene_LF2_split[['mutationinformation', 'lineage_corrupt_list', 'lineage_corrupt_ucount']]
|
|
gene_LF2_split[['mutationinformation', 'lineage_ucount', 'lineage_corrupt_ucount']]
|
|
|
|
#%% AF: calculate AF for each mutation
|
|
#1) calculate no. of unique ids
|
|
gene_LF2['id_ucount']/total_id_ucount
|
|
|
|
#%% DM OM labels
|
|
# COPY mutation_info_labels column
|
|
gene_LF2['mutation_info_labels_orig'] = gene_LF2['mutation_info_labels']
|
|
|
|
# Convert DM/OM labels to numeric
|
|
dm_om_map = {'DM': 1, 'OM': 0} # pnca, OM is minority, other genes: DM is minority
|
|
gene_LF2['dm_om_numeric'] = gene_LF2['mutation_info_labels'].map(dm_om_map)
|
|
# sanity check
|
|
gene_LF2['dm_om_numeric'].value_counts()
|
|
gene_LF2['mutation_info_labels'].value_counts()
|
|
|
|
# Convert drtype column to numeric
|
|
drtype_map = {'XDR': 5
|
|
, 'Pre-XDR': 4
|
|
, 'MDR': 3
|
|
, 'Pre-MDR': 2
|
|
, 'Other': 1
|
|
, 'Sensitive': 0}
|
|
|
|
gene_LF2['drtype_numeric'] = gene_LF2['drtype'].map(drtype_map)
|
|
|
|
# COPY dst column
|
|
gene_LF2['dst'] = gene_LF2[drug] # to allow cross checking
|
|
gene_LF2['dst_multimode'] = gene_LF2[drug]
|
|
|
|
# sanity check
|
|
gene_LF2[drug].value_counts()
|
|
gene_LF2['dst_multimode'].value_counts()
|
|
|
|
gene_LF2[drug].isnull().sum()
|
|
gene_LF2['dst_multimode'].isnull().sum()
|
|
|
|
gene_LF2['mutationinformation'].value_counts()
|
|
#gene_LF2.C.isnull().groupby([df['A'],df['B']]).sum().astype(int).reset_index(name='count')
|
|
gene_LF2[drug].isnull().groupby(gene_LF2['mutationinformation']).sum()
|
|
|
|
# GOAL is to populate na in the dst column from the count of the dm_om_numeric column
|
|
gene_LF2['dst_multimode'].isnull().groupby(gene_LF2['mutationinformation']).sum()
|
|
|
|
gene_LF2['mutationinformation']
|
|
|
|
#%% Recalculating dst: my gene_LF2
|
|
#------------------------------
|
|
# Revised dst: max(multimode)
|
|
#------------------------------
|
|
# For each mutation, generate the revised dst which is the mode of dm_om_numeric
|
|
# PROBLEM: Returns the smallest of the two when bimodal, and fails when all equally likely
|
|
# SOLUTION: Using max of the 'dst_noNA' column
|
|
#gene_LF22.groupby('mutationinformation')['dm_om_numeric'].agg(multimode)
|
|
|
|
# Get multimode for dm_om_numeric column
|
|
dm_om_multimode = gene_LF2.groupby('mutationinformation')['dm_om_numeric'].agg(multimode)
|
|
#dm_om_multimode
|
|
|
|
# Fill using multimode ONLY where NA in dst_multimode column
|
|
#gene_LF22['dst_multimode'] = gene_LF22['dst_multimode'].fillna(dm_om_multimode)
|
|
gene_LF2['dst_multimode'] = gene_LF2['dst_multimode'].fillna(dm_om_multimode)
|
|
|
|
# gene_LF22['dst_multimode']
|
|
|
|
# Now get the max from multimode
|
|
gene_LF22['dst_noNA'] = gene_LF2['dst_multimode'].apply(lambda x: np.nanmax(x))
|
|
print(gene_LF2)
|
|
|
|
# Finally created a revised dst with the max from the multimode
|
|
gene_LF22['dst_mode'] = gene_LF2.groupby('mutationinformation')['dst_noNA'].max()
|
|
#==============================================================================
|
|
#%% Recalculating drtype: my gene_LF2
|
|
#--------------------------------
|
|
# drtype: ALL values:
|
|
# numeric and names in an array
|
|
#--------------------------------
|
|
gene_LF2['drtype_all_vals'] = gene_LF2['drtype_numeric']
|
|
gene_LF2['drtype_all_names'] = gene_LF2['drtype']
|
|
|
|
# example: https://stackoverflow.com/questions/55125680/pandas-get-all-groupby-values-in-an-array
|
|
# print(df.groupby('key').gene_LF2.apply(list).reset_index()) # my use case, don't need the reset_index()
|
|
gene_LF2['drtype_all_vals'] = gene_LF2.groupby('mutationinformation').drtype_all_vals.apply(list)
|
|
gene_LF2['drtype_all_names'] = gene_LF2.groupby('mutationinformation').drtype_all_names.apply(list)
|
|
|
|
#---------------------------------
|
|
# Revised drtype: max(Multimode)
|
|
#--------------------------------
|
|
gene_LF2['drtype_multimode'] = gene_LF2.groupby(['mutationinformation'])['drtype_numeric'].agg(multimode)
|
|
gene_LF2['drtype_multimode']
|
|
|
|
# Now get the max from multimode
|
|
gene_LF2['drtype_mode'] = gene_LF2['drtype_multimode'].apply(lambda x: np.nanmax(x))
|
|
gene_LF2.head()
|
|
|
|
#----------------------
|
|
# Revised drtype: Max
|
|
#----------------------
|
|
gene_LF2.head()
|
|
gene_LF2['drtype_max'] = gene_LF2.groupby(['mutationinformation'])['drtype_numeric'].max()
|
|
#gene_LF2 = gene_LF22.reset_index()
|
|
gene_LF2.head()
|
|
|
|
#%% Finally reset index
|
|
gene_LF2 = gene_LF2.reset_index()
|
|
#==============================================================================
|
|
#---------------------------------------
|
|
# Create revised mutation_info_column
|
|
#---------------------------------------
|
|
gene_LF2['dst_mode'].value_counts()
|
|
gene_LF2[drug].value_counts()
|
|
|
|
# note this is overriding, since downstream depends on it
|
|
# make a copy you if you need to keep that
|
|
gene_LF2['mutation_info_labels_orig'] = gene_LF2['mutation_info_labels']
|
|
gene_LF2['mutation_info_labels'] = gene_LF2['dst_mode'].map({1: 'DM'
|
|
, 0: 'OM'})
|
|
gene_LF2['mutation_info_labels_orig'].value_counts()
|
|
gene_LF2['mutation_info_labels'].value_counts()
|
|
#==============================================================================
|
|
# sanity check
|
|
if (all(gene_LF2['mutation'] == gene_LF2['mutationinformation'])):
|
|
print('\nPass: Mutationinformation check successful')
|
|
else:
|
|
sys.exit('\nERROR: mutationin cross checks failed. Please check your group_by() aggregate functions')
|
|
|
|
# Drop mutation column
|
|
gene_LF2.drop(['mutation'], axis=1, inplace=True)
|
|
#%% subset: equivalent of merged_df3?
|
|
# https://stackoverflow.com/questions/39900061/sort-lists-in-a-pandas-gene_LF2frame-column
|
|
|
|
# result = gene_LF2['dst_multimode'].sort_values().apply(lambda x: sorted(x))
|
|
# newdf = pd.gene_LF2Frame({'dst_multimode': Series(list(set(result['a'].apply(tuple))))})
|
|
# newdf.sort_values(by='a')
|
|
|
|
# gene_LF2['dst_multimode'].value_counts()
|
|
# gene_LF2.sort_values(['dst_multimode'], ascending=False)
|
|
|
|
#gene_LF2_df3 = gene_LF2.drop_duplicates(['mutationinformation'])
|
|
#gene_LF2_df3_v2 = gene_LF2.drop_duplicates(['mutationinformation'])
|
|
#all(gene_LF2_df3 == gene_LF2_df3_v2)
|
|
#%%
|