updated combining df scripts for duet & lig
This commit is contained in:
parent
c184841951
commit
96ebb85069
3 changed files with 458 additions and 342 deletions
|
|
@ -1,9 +1,9 @@
|
|||
#============================================
|
||||
#########################################################
|
||||
# TASK: To calculate Allele Frequency and
|
||||
# Odds Ratio from master data
|
||||
# and add the calculated params to meta_data extracted from
|
||||
# pnca_data_extraction.py
|
||||
#===========================================
|
||||
# data_extraction.py
|
||||
#########################################################
|
||||
getwd()
|
||||
setwd('~/git/LSHTM_analysis/meta_data_analysis')
|
||||
getwd()
|
||||
|
|
@ -14,9 +14,9 @@ gene = 'pncA'
|
|||
gene_match = paste0(gene,'_p.')
|
||||
cat(gene_match)
|
||||
|
||||
#=======
|
||||
# input dir
|
||||
#=======
|
||||
#===========
|
||||
# input
|
||||
#===========
|
||||
# infile1: Raw data
|
||||
#indir = 'git/Data/pyrazinamide/input/original'
|
||||
indir = paste0('~/git/Data')
|
||||
|
|
@ -24,27 +24,26 @@ in_filename = 'original_tanushree_data_v2.csv'
|
|||
infile = paste0(indir, '/', in_filename)
|
||||
cat(paste0('Reading infile1: raw data', ' ', infile) )
|
||||
|
||||
# infile2: gene associated meta data file to extract valid snps and add calcs to
|
||||
# filename: outfile3 from data_extraction.py
|
||||
# infile2: gene associated meta data file to extract valid snps and add calcs to.
|
||||
# This is outfile3 from data_extraction.py
|
||||
indir_metadata = paste0('~/git/Data', '/', drug, '/', 'output')
|
||||
in_filename_metadata = 'pnca_metadata.csv'
|
||||
infile_metadata = paste0(indir_metadata, '/', in_filename_metadata)
|
||||
cat(paste0('Reading infile2: gene associated metadata:', infile_metadata))
|
||||
|
||||
#=========
|
||||
# output dir
|
||||
#=========
|
||||
#===========
|
||||
# output
|
||||
#===========
|
||||
# outdir = 'git/Data/pyrazinamide/output'
|
||||
outdir = paste0('~/git/Data', '/', drug, '/', 'output')
|
||||
out_filename = paste0(tolower(gene),'_', 'meta_data_with_AFandOR.csv')
|
||||
outfile = paste0(outdir, '/', out_filename)
|
||||
cat(paste0('Output file with full path:', outfile))
|
||||
|
||||
#%% end of variable assignment for input and output files
|
||||
|
||||
#===============
|
||||
# Step 1: Read master/raw data stored in Data/
|
||||
#===============
|
||||
#########################################################
|
||||
# 1: Read master/raw data stored in Data/
|
||||
#########################################################
|
||||
raw_data_all = read.csv(infile, stringsAsFactors = F)
|
||||
|
||||
raw_data = raw_data_all[,c("id"
|
||||
|
|
@ -55,9 +54,9 @@ rm(raw_data_all)
|
|||
|
||||
rm(indir, in_filename, infile)
|
||||
|
||||
#####
|
||||
#===========
|
||||
# 1a: exclude na
|
||||
#####
|
||||
#===========
|
||||
raw_data = raw_data[!is.na(raw_data$pyrazinamide),]
|
||||
|
||||
total_samples = length(unique(raw_data$id))
|
||||
|
|
@ -66,16 +65,16 @@ cat(paste0('Total samples without NA in', ' ', drug, 'is:', total_samples))
|
|||
# sanity check: should be true
|
||||
is.numeric(total_samples)
|
||||
|
||||
#####
|
||||
#===========
|
||||
# 1b: combine the two mutation columns
|
||||
#####
|
||||
#===========
|
||||
raw_data$all_mutations_pyrazinamide = paste(raw_data$dr_mutations_pyrazinamide
|
||||
, raw_data$other_mutations_pyrazinamide)
|
||||
head(raw_data$all_mutations_pyrazinamide)
|
||||
|
||||
#####
|
||||
#===========
|
||||
# 1c: create yet another column that contains all the mutations but in lower case
|
||||
#####
|
||||
#===========
|
||||
raw_data$all_muts_pnca = tolower(raw_data$all_mutations_pyrazinamide)
|
||||
|
||||
# sanity checks
|
||||
|
|
@ -104,9 +103,10 @@ table(mut, dst)
|
|||
#fisher.test(table(mut, dst))
|
||||
#table(mut)
|
||||
|
||||
#===============
|
||||
# Step 2: Read valid snps for which OR can be calculated (infile_comp_snps.csv)
|
||||
#===============
|
||||
#########################################################
|
||||
# 2: Read valid snps for which OR
|
||||
# can be calculated (infile_comp_snps.csv)
|
||||
#########################################################
|
||||
cat(paste0('Reading metadata infile:', infile_metadata))
|
||||
|
||||
pnca_metadata = read.csv(infile_metadata
|
||||
|
|
@ -188,7 +188,7 @@ hist(log(ors)
|
|||
, breaks = 100
|
||||
)
|
||||
|
||||
# FIXME: could be good to add a sanity check
|
||||
# sanity check
|
||||
if (table(names(ors) == names(pvals)) & table(names(ors) == names(afs)) & table(names(pvals) == names(afs)) == length(pnca_snps_unique)){
|
||||
cat('PASS: names of ors, pvals and afs match: proceed with combining into a single df')
|
||||
} else{
|
||||
|
|
@ -214,9 +214,9 @@ if (table(rownames(comb_AF_and_OR) == comb_AF_and_OR$mutation)){
|
|||
cat('FAIL: rownames and mutation col values mismatch')
|
||||
}
|
||||
|
||||
############
|
||||
# Merge 1: combine meta data file with calculated num params
|
||||
###########
|
||||
#########################################################
|
||||
# 3: Merge meta data file + calculated num params
|
||||
#########################################################
|
||||
df1 = pnca_metadata
|
||||
df2 = comb_AF_and_OR
|
||||
|
||||
|
|
@ -254,7 +254,7 @@ if ( identical(na_count[[length(na_count)]], na_count[[length(na_count)-1]], na_
|
|||
cat('PASS: No. of NAs for OR, AF and Pvals are equal as expected',
|
||||
'\nNo. of NA: ', na_count[[length(na_count)]])
|
||||
} else {
|
||||
cat('FAIl: No. of NAs for OR, AF and Pvals mismatch')
|
||||
cat('FAIL: No. of NAs for OR, AF and Pvals mismatch')
|
||||
}
|
||||
|
||||
# reassign custom colnames
|
||||
|
|
@ -289,6 +289,7 @@ cat(paste0('Added', ' ', ncol_added, ' more cols to merged_df:'
|
|||
, '\nno. of cols in merged_df now: ', ncol(merged_df)))
|
||||
|
||||
#%% write file out: pnca_meta_data_with_AFandOR
|
||||
#*********************************************
|
||||
cat(paste0('writing output file: '
|
||||
, '\nFilename: ', out_filename
|
||||
, '\nPath:', outdir))
|
||||
|
|
@ -300,6 +301,98 @@ cat(paste0('Finished writing:'
|
|||
, out_filename
|
||||
, '\nNo. of rows: ', nrow(merged_df)
|
||||
, '\nNo. of cols: ', ncol(merged_df)))
|
||||
|
||||
#************************************************
|
||||
cat('======================================================================')
|
||||
rm(out_filename)
|
||||
cat('End of script: calculated AF, OR, pvalues and saved file')
|
||||
# End of script
|
||||
#%%
|
||||
# sanity check: Count NA in these four cols.
|
||||
# However these need to be numeric else these
|
||||
# will be misleading when counting NAs (i.e retrun 0)
|
||||
#is.numeric(meta_with_afor$OR)
|
||||
na_var = c('AF', 'OR', 'pvalue', 'logor', 'neglog10pvalue')
|
||||
|
||||
# loop through these vars and check if these are numeric.
|
||||
# if not, then convert to numeric
|
||||
check_all = NULL
|
||||
|
||||
for (i in na_var){
|
||||
# cat(i)
|
||||
check0 = is.numeric(meta_with_afor[,i])
|
||||
if (check0) {
|
||||
check_all = c(check0, check_all)
|
||||
cat('These are all numeric cols')
|
||||
} else{
|
||||
cat('First converting to numeric')
|
||||
check0 = as.numeric(meta_with_afor[,i])
|
||||
check_all = c(check0, check_all)
|
||||
}
|
||||
}
|
||||
|
||||
# count na now that the respective cols are numeric
|
||||
na_count = sapply(meta_with_afor, function(y) sum(length(which(is.na(y))))); na_count
|
||||
str(na_count)
|
||||
|
||||
# extract how many NAs:
|
||||
# should be all TRUE
|
||||
# should be a single number since
|
||||
# all the cols should have 'equal' and 'same' no. of NAs
|
||||
# compare if the No of 'NA' are the same for all these cols
|
||||
na_len = NULL
|
||||
for (i in na_var){
|
||||
temp = na_count[[i]]
|
||||
na_len = c(na_len, temp)
|
||||
}
|
||||
|
||||
cat('Checking how many NAs and if these are identical for the selected cols:')
|
||||
my_nrows = NULL
|
||||
for ( i in 1: (length(na_len)-1) ){
|
||||
# cat(compare(na_len[i]), na_len[i+1])
|
||||
c = compare(na_len[i], na_len[i+1])
|
||||
if ( c$result ) {
|
||||
cat('PASS: No. of NAa in selected cols are identical')
|
||||
my_nrows = na_len[i] }
|
||||
else {
|
||||
cat('FAIL: No. of NAa in selected cols mismatch')
|
||||
}
|
||||
}
|
||||
|
||||
cat('No. of NAs in each of the selected cols: ', my_nrows)
|
||||
|
||||
# yet more sanity checks:
|
||||
cat('Check whether the ', my_nrows, 'indices are indeed the same')
|
||||
|
||||
#which(is.na(meta_with_afor$OR))
|
||||
|
||||
# initialise an empty df with nrows as extracted above
|
||||
na_count_df = data.frame(matrix(vector(mode = 'numeric'
|
||||
# , length = length(na_var)
|
||||
)
|
||||
, nrow = my_nrows
|
||||
# , ncol = length(na_var)
|
||||
))
|
||||
|
||||
# populate the df with the indices of the cols that are NA
|
||||
for (i in na_var){
|
||||
cat(i)
|
||||
na_i = which(is.na(meta_with_afor[i]))
|
||||
na_count_df = cbind(na_count_df, na_i)
|
||||
colnames(na_count_df)[which(na_var == i)] <- i
|
||||
}
|
||||
|
||||
# Now compare these indices to ensure these are the same
|
||||
check2 = NULL
|
||||
for ( i in 1: ( length(na_count_df)-1 ) ) {
|
||||
# cat(na_count_df[i] == na_count_df[i+1])
|
||||
check_all = identical(na_count_df[[i]], na_count_df[[i+1]])
|
||||
check2 = c(check_all, check2)
|
||||
if ( all(check2) ) {
|
||||
cat('PASS: The indices for AF, OR, etc are all the same\n')
|
||||
} else {
|
||||
cat ('FAIL: Please check indices which are NA')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue