updated pnca_extraction and AF_OR calcs
This commit is contained in:
parent
53d19d5dd8
commit
f686563c98
4 changed files with 195 additions and 699 deletions
|
|
@ -1,16 +1,59 @@
|
|||
#============================================
|
||||
# TASK: to calculate allele frequency and OR
|
||||
# on master data
|
||||
#===========================================
|
||||
homedir = '~'
|
||||
getwd()
|
||||
setwd("/git/github/git/LSHTM_analysis/meta_data_analysis")
|
||||
#setwd('~/git/LSHTM_analysis/meta_data_analysis')
|
||||
setwd(paste0(homedir, '/', 'git/LSHTM_analysis/meta_data_analysis'))
|
||||
getwd()
|
||||
|
||||
#%% variable assignment: input and output paths & filenames
|
||||
drug = 'pyrazinamide'
|
||||
gene = 'pncA'
|
||||
gene_match = paste0(gene,'_p.')
|
||||
print(gene_match)
|
||||
|
||||
#=======
|
||||
# input dir
|
||||
#=======
|
||||
# file1: Raw data
|
||||
#indir = 'git/Data/pyrazinamide/input/original'
|
||||
indir = paste0('git/Data', '/', drug, '/', 'input/original')
|
||||
in_filename = 'original_tanushree_data_v2.csv'
|
||||
infile = paste0(homedir, '/', indir, '/', in_filename)
|
||||
print(paste0('Reading infile:', ' ', infile) )
|
||||
|
||||
# file2: file to extract valid snps and add calcs to: pnca_metadata.csv {outfile3 from data extraction script}
|
||||
indir_metadata = paste0('git/Data', '/', drug, '/', 'output')
|
||||
in_filename_metadata = 'pnca_metadata.csv'
|
||||
infile_metadata = paste0(homedir, '/', indir_metadata, '/', in_filename_metadata)
|
||||
print(paste0('Reading metadata infile:', infile_metadata))
|
||||
|
||||
#=========
|
||||
# output dir
|
||||
#=========
|
||||
# output filename in respective section at the time of outputting files
|
||||
#outdir = 'git/Data/pyrazinamide/output'
|
||||
outdir = paste0('git/Data', '/', drug, '/', 'output')
|
||||
out_filename = paste0(tolower(gene),'_', 'meta_data_with_AFandOR.csv')
|
||||
outfile = paste0(homedir, '/', outdir, '/', out_filename)
|
||||
print(paste0('Output file with full path:', outfile))
|
||||
|
||||
#%% end of variable assignment for input and output files
|
||||
|
||||
#===============
|
||||
# Step 1: read GWAS raw data stored in Data_original/
|
||||
#===============
|
||||
infile = read.csv(file.choose(), stringsAsFactors = F)
|
||||
# Step 1: Read master/raw data stored in Data/
|
||||
#===============
|
||||
raw_data_all = read.csv(infile, stringsAsFactors = F)
|
||||
|
||||
raw_data = infile[,c("id"
|
||||
raw_data = raw_data_all[,c("id"
|
||||
, "pyrazinamide"
|
||||
, "dr_mutations_pyrazinamide"
|
||||
, "other_mutations_pyrazinamide")]
|
||||
rm(raw_data_all)
|
||||
|
||||
rm(indir, in_filename, infile)
|
||||
|
||||
#####
|
||||
# 1a: exclude na
|
||||
|
|
@ -18,7 +61,7 @@ raw_data = infile[,c("id"
|
|||
raw_data = raw_data[!is.na(raw_data$pyrazinamide),]
|
||||
|
||||
total_samples = length(unique(raw_data$id))
|
||||
print(total_samples)
|
||||
print(paste0('Total samples without NA in', ' ', drug, 'is:', total_samples))
|
||||
|
||||
# sanity check: should be true
|
||||
is.numeric(total_samples)
|
||||
|
|
@ -36,10 +79,15 @@ head(raw_data$all_mutations_pyrazinamide)
|
|||
raw_data$all_muts_pnca = tolower(raw_data$all_mutations_pyrazinamide)
|
||||
|
||||
# sanity checks
|
||||
table(grepl("pnca_p",raw_data$all_muts_pnca))
|
||||
#table(grepl("pnca_p",raw_data$all_muts_pnca))
|
||||
print(paste0('converting gene match:', gene_match, ' ', 'to lowercase'))
|
||||
gene_match = tolower(gene_match)
|
||||
|
||||
table(grepl(gene_match,raw_data$all_muts_pnca))
|
||||
|
||||
# sanity check: should be TRUE
|
||||
sum(table(grepl("pnca_p",raw_data$all_muts_pnca))) == total_samples
|
||||
#sum(table(grepl("pnca_p",raw_data$all_muts_pnca))) == total_samples
|
||||
sum(table(grepl(gene_match,raw_data$all_muts_pnca))) == total_samples
|
||||
|
||||
# set up variables: can be used for logistic regression as well
|
||||
i = "pnca_p.ala134gly" # has a NA, should NOT exist
|
||||
|
|
@ -56,17 +104,40 @@ table(mut, dst)
|
|||
#fisher.test(table(mut, dst))
|
||||
#table(mut)
|
||||
|
||||
###### read list of muts to calculate OR for (fname3 from pnca_data_extraction.py)
|
||||
pnca_snps_or = read.csv(file.choose()
|
||||
#===============
|
||||
# Step 2: Read valid snps for which OR can be calculated (infile_comp_snps.csv)
|
||||
#===============
|
||||
print(paste0('Reading metadata infile:', infile_metadata))
|
||||
|
||||
pnca_metadata = read.csv(infile_metadata
|
||||
# , file.choose()
|
||||
, stringsAsFactors = F
|
||||
, header = T)
|
||||
|
||||
# extract unique snps to iterate over for AF and OR calcs
|
||||
# total no of unique snps
|
||||
# AF and OR calculations
|
||||
|
||||
# clear variables
|
||||
rm(homedir, in_filename, indir, infile)
|
||||
rm(indir_metadata, infile_metadata, in_filename_metadata)
|
||||
|
||||
# count na in pyrazinamide column
|
||||
tot_pza_na = sum(is.na(pnca_metadata$pyrazinamide))
|
||||
expected_rows = nrow(pnca_metadata) - tot_pza_na
|
||||
|
||||
# drop na from the pyrazinamide colum
|
||||
pnca_snps_or = pnca_metadata[!is.na(pnca_metadata$pyrazinamide),]
|
||||
|
||||
# sanity check
|
||||
if(nrow(pnca_snps_or) == expected_rows){
|
||||
print('PASS: no. of rows match with expected_rows')
|
||||
} else{
|
||||
print('FAIL: nrows mismatch.')
|
||||
}
|
||||
|
||||
# extract unique snps to iterate over for AF and OR calcs
|
||||
pnca_snps_unique = unique(pnca_snps_or$mutation)
|
||||
|
||||
print(paste0('Total no. of distinct comp snps to perform OR calcs: ', length(pnca_snps_unique)))
|
||||
|
||||
# Define OR function
|
||||
x = as.numeric(mut)
|
||||
y = dst
|
||||
|
|
@ -111,131 +182,106 @@ afs['pnca_p.trp68gly']
|
|||
afs['pnca_p.gln10pro']
|
||||
afs['pnca_p.leu4ser']
|
||||
|
||||
#plot(density(log(ors)))
|
||||
#plot(-log10(pvals))
|
||||
#hist(log(ors)
|
||||
# ,breaks = 100
|
||||
# )
|
||||
plot(density(log(ors)))
|
||||
plot(-log10(pvals))
|
||||
hist(log(ors)
|
||||
, breaks = 100
|
||||
)
|
||||
|
||||
# subset df cols to add to the calc param df
|
||||
pnca_snps_cols = pnca_snps_or[c('mutation_info', 'mutation', 'Mutationinformation')]
|
||||
pnca_snps_cols = pnca_snps_cols[!duplicated(pnca_snps_cols$mutation),]
|
||||
|
||||
rownames(pnca_snps_cols) = pnca_snps_cols$mutation
|
||||
head(rownames(pnca_snps_cols))
|
||||
#snps_with_AF_and_OR
|
||||
# FIXME: could be good to add a sanity check
|
||||
if (table(names(ors) == names(pvals)) & table(names(ors) == names(afs)) & table(names(pvals) == names(afs)) == length(pnca_snps_unique)){
|
||||
print('PASS: names of ors, pvals and afs match: proceed with combining into a single df')
|
||||
} else{
|
||||
print('FAIL: names of ors, pvals and afs mismatch')
|
||||
}
|
||||
|
||||
# combine
|
||||
comb_AF_and_OR = data.frame(ors, pvals, afs)
|
||||
head(rownames(comb_AF_and_OR))
|
||||
|
||||
# sanity checks: should be the same
|
||||
dim(comb_AF_and_OR); dim(pnca_snps_cols)
|
||||
table(rownames(comb_AF_and_OR)%in%rownames(pnca_snps_cols))
|
||||
|
||||
table(rownames(pnca_snps_cols)%in%rownames(comb_AF_and_OR))
|
||||
|
||||
# merge the above two df whose dim you checked
|
||||
snps_with_AF_and_OR = merge(comb_AF_and_OR, pnca_snps_cols
|
||||
, by = "row.names"
|
||||
# , all.x = T
|
||||
)
|
||||
|
||||
#rm(pnca_snps_cols, pnca_snps_or, raw_data)
|
||||
|
||||
#===============
|
||||
# Step 3: Read data file where you will add the calculated OR
|
||||
# Note: this is the big file with one-many relationship between snps and lineages
|
||||
# i.e fname4 from 'pnca_extraction.py'
|
||||
#===============
|
||||
my_data = read.csv(file.choose()
|
||||
, row.names = 1
|
||||
, stringsAsFactors = FALSE)
|
||||
|
||||
head(my_data)
|
||||
length(unique(my_data$id))
|
||||
|
||||
# check if first col is 'id': should be TRUE
|
||||
colnames(my_data)[1] == 'id'
|
||||
# add rownames of comb_AF_and_OR as an extra column 'mutation' to allow merging based on this column
|
||||
comb_AF_and_OR$mutation = rownames(comb_AF_and_OR)
|
||||
|
||||
# sanity check
|
||||
head(my_data$mutation)
|
||||
if (table(rownames(comb_AF_and_OR) == comb_AF_and_OR$mutation)){
|
||||
print('PASS: rownames and mutaion col values match')
|
||||
}else{
|
||||
print('FAIL: rownames and mutation col values mismatch')
|
||||
}
|
||||
|
||||
# FILES TO MERGE:
|
||||
# comb_AF_and_OR: file containing OR
|
||||
# my_data = big meta data file
|
||||
# linking column: mutation
|
||||
############
|
||||
# Merge 1:
|
||||
###########
|
||||
df1 = pnca_metadata
|
||||
df2 = comb_AF_and_OR
|
||||
|
||||
head(my_data)
|
||||
merged_df = merge(my_data # big file
|
||||
, snps_with_AF_and_OR # small (afor file)
|
||||
head(df1$mutation); head(df2$mutation)
|
||||
|
||||
# FIXME: newlines
|
||||
print(paste0('merging two dfs: '
|
||||
,'\ndf1 (big df i.e. meta data) nrows: ', nrow(df1)
|
||||
,'\ndf2 (small df i.e af, or, pval) nrows: ', nrow(df2)
|
||||
, 'expected rows in merged df: ', nrow(df1), 'expected cols in merged_df: ', (ncol(df1) + ncol(df2) - 1)))
|
||||
|
||||
merged_df = merge(df1 # big file
|
||||
, df2 # small (afor file)
|
||||
, by = "mutation"
|
||||
, all.x = T) # because you want all the entries of the meta data
|
||||
|
||||
# sanity checks: should be True
|
||||
# FIXME: I have checked this manually, but make it so it is a pass or a fail!
|
||||
comb_AF_and_OR[rownames(comb_AF_and_OR) == "pnca_p.gln10pro",]$ors
|
||||
merged_df[merged_df$Mutationinformation.x == "Q10P",]$ors
|
||||
# sanity check
|
||||
if(ncol(merged_df) == (ncol(df1) + ncol(df2) - 1)){
|
||||
print(paste0('PASS: no. of cols is as expected: ', ncol(merged_df)))
|
||||
} else{
|
||||
print('FAIL: no.of cols mistmatch')
|
||||
}
|
||||
|
||||
merged_df[merged_df$Mutationinformation.x == "Q10P",]
|
||||
|
||||
# sanity check: very important!
|
||||
colnames(merged_df)
|
||||
|
||||
table(merged_df$mutation_info.x == merged_df$mutation_info.y)
|
||||
|
||||
#FIXME: what happened to other 7 and FALSE
|
||||
table(merged_df$Mutationinformation.x == merged_df$Mutationinformation.y)
|
||||
|
||||
# problem
|
||||
identical(merged_df$Mutationinformation.x, merged_df$Mutationinformation.y)
|
||||
|
||||
#merged_df[merged_df$Mutationinformation.x != merged_df$Mutationinformation.y,]
|
||||
|
||||
#throw away the y because that is a smaller df
|
||||
d1 = which(colnames(merged_df) == "mutation_info.y") #21
|
||||
d2 = which(colnames(merged_df) == "Mutationinformation.y") #22
|
||||
|
||||
merged_df2 = merged_df[-c(d1, d2)] #3093 20
|
||||
colnames(merged_df2)
|
||||
|
||||
# rename cols
|
||||
colnames(merged_df2)[colnames(merged_df2)== "mutation_info.x"] <- "mutation_info"
|
||||
colnames(merged_df2)[colnames(merged_df2)== "Mutationinformation.x"] <- "Mutationinformation"
|
||||
|
||||
colnames(merged_df2)
|
||||
|
||||
# should be 0
|
||||
sum(is.na(merged_df2$Mutationinformation))
|
||||
# quick check
|
||||
i = "pnca_p.ala134gly" # has all NAs in pyrazinamide, should be NA in ors, etc.
|
||||
merged_df[merged_df$mutation == i,]
|
||||
|
||||
# count na in each column
|
||||
na_count = sapply(merged_df2, function(y) sum(length(which(is.na(y))))); na_count
|
||||
na_count = sapply(merged_df, function(y) sum(length(which(is.na(y))))); na_count
|
||||
# only some or and Af should be NA
|
||||
#Row.names ors pvals afs
|
||||
#81 81 81 81
|
||||
#63 63 63 63
|
||||
|
||||
# reassign custom colnames
|
||||
colnames(merged_df)[colnames(merged_df)== "ors"] <- "OR"
|
||||
colnames(merged_df)[colnames(merged_df)== "afs"] <- "AF"
|
||||
colnames(merged_df)[colnames(merged_df)== "pvals"] <- "pvalue"
|
||||
|
||||
colnames(merged_df2)[colnames(merged_df2)== "ors"] <- "OR"
|
||||
colnames(merged_df2)[colnames(merged_df2)== "afs"] <- "AF"
|
||||
colnames(merged_df2)[colnames(merged_df2)== "pvals"] <- "pvalue"
|
||||
|
||||
colnames(merged_df2)
|
||||
colnames(merged_df)
|
||||
|
||||
# add log OR and neglog pvalue
|
||||
merged_df2$logor = log(merged_df2$OR)
|
||||
is.numeric(merged_df2$logor)
|
||||
merged_df$logor = log(merged_df$OR)
|
||||
is.numeric(merged_df$logor)
|
||||
|
||||
merged_df2$neglog10pvalue = -log10(merged_df2$pvalue)
|
||||
is.numeric(merged_df2$neglog10pvalue)
|
||||
merged_df$neglog10pvalue = -log10(merged_df$pvalue)
|
||||
is.numeric(merged_df$neglog10pvalue)
|
||||
|
||||
# write file out
|
||||
#write.csv(merged_df, "../Data/meta_data_with_AFandOR_JP_TT.csv")
|
||||
merged_df$AF_percent = merged_df$AF*100
|
||||
is.numeric(merged_df$AF_percent)
|
||||
|
||||
# define output variables
|
||||
drug = 'pyrazinamide'
|
||||
out_dir = paste0("../mcsm_analysis/",drug,"/Data/")
|
||||
outFile = "meta_data_with_AFandOR.csv"
|
||||
output_filename = paste0(outdir, outFile)
|
||||
# check AFs
|
||||
#i = 'pnca_p.trp68gly'
|
||||
i = 'pnca_p.gln10pro'
|
||||
#i = 'pnca_p.leu4ser'
|
||||
merged_df[merged_df$mutation == i,]
|
||||
|
||||
write.csv(merged_df2, output_filename
|
||||
# FIXME: harcoding (beware!), NOT FATAL though!
|
||||
ncol_added = 3
|
||||
|
||||
print(paste0('Added', ncol_added, ' ', 'more cols to merged_df i.e log10 OR and -log10 P-val: '
|
||||
, 'no. of cols in merged_df now: ', ncol(merged_df)))
|
||||
|
||||
#%% write file out: pnca_meta_data_with_AFandOR
|
||||
print(paste0('writing output file in: '
|
||||
, 'Filename: ', out_filename
|
||||
, 'Path:', outdir))
|
||||
|
||||
write.csv(merged_df, outfile
|
||||
, row.names = F)
|
||||
|
||||
print(paste0('Finished writing:', out_filename, '\nExpected no. of cols:', ncol(merged_df)))
|
||||
print('======================================================================')
|
||||
rm(out_filename)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue