added sample test data for processing to get correct annotations

test_data/processing.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Mar 24 15:01:59 2022
+
+@author: tanu
+"""
+import sys, os
+import pandas as pd
+import numpy as np
+from statistics import mean, median, mode
+#from statistics import multimode
+from collections import Counter
+import math
+
+# https://stackoverflow.com/questions/43321455/pandas-count-null-values-in-a-groupby-function
+#https://stackoverflow.com/questions/33457191/python-pandas-dataframe-fill-nans-with-a-conditional-mean
+#%%
+drug = "pyrazinamide"
+
+data = pd.read_csv("/home/tanu/git/ML_AI_training/test_data/sample_data.csv")
+data.columns
+
+# Convert DM/OM labels to numeric
+dm_om_map = {'DM': 1, 'OM': 0} # pnca, OM is minority, other genes: DM is minority
+data['dm_om_numeric'] = data['mutation_info_labels'].map(dm_om_map)
+# sanity check
+data['dm_om_numeric'].value_counts()
+data['mutation_info_labels'].value_counts()
+
+# COPY dst column
+data['dst'] = data[drug]
+# sanity check
+data[drug].value_counts()
+data[drug].isnull().sum()
+
+data['dst'].value_counts()
+data['dst'].isnull().sum()
+
+data['mutationinformation'].value_counts()
+#data.C.isnull().groupby([df['A'],df['B']]).sum().astype(int).reset_index(name='count')
+data[drug].isnull().groupby(data['mutationinformation']).sum()
+                             
+# GOAL is to populate na in the dst column from the count of the dm_om_numeric column                          
+data['dst'].isnull().groupby(data['mutationinformation']).sum()                       
+
+
+# round up
+int(math.ceil(mean(foo)))  
+#https://stackoverflow.com/questions/33457191/python-pandas-dataframe-fill-nans-with-a-conditional-mean
+#FIXME
+# STAGE 1: replace mean with  Max(multimode), atm it is MEAN
+#na_val = data.groupby(data['mutationinformation'])['dst'].mean()
+
+data['dst'] = data['dst'].fillna(data.groupby('mutationinformation')['dst'].transform('mean'))
+
+# FIXME
+#STAGE 2: Fill TRUE nan with DM.OM column value, atm it is MEAN
+data['dst2'] = data['dst'].fillna(data.groupby('mutationinformation')['dm_om_numeric'].transform('mean'))
+data['dst2'] = data['dst'].fillna(data.groupby('mutationinformation').transform(['dm_om_numeric']))
+