turned combining_dfs_plotting.R to a function and moved old script to redundant
This commit is contained in:
parent
cd5cbce3a0
commit
ea79f3b3c7
3 changed files with 327 additions and 883 deletions
|
|
@ -68,18 +68,18 @@ import_dirs(drug, gene)
|
||||||
# my_df_u_lig
|
# my_df_u_lig
|
||||||
# dup_muts
|
# dup_muts
|
||||||
#***********************************
|
#***********************************
|
||||||
#infile = "/home/tanu/git/Data/streptomycin/output/gid_comb_stab_struc_params.csv"
|
#infile_params = "/home/tanu/git/Data/streptomycin/output/gid_comb_stab_struc_params.csv"
|
||||||
|
|
||||||
#if (!exists("infile") && exists("gene")){
|
if (!exists("infile_params") && exists("gene")){
|
||||||
if (!is.character(infile) && exists("gene")){
|
#if (!is.character(infile_params) && exists("gene")){
|
||||||
#in_filename_params = paste0(tolower(gene), "_all_params.csv")
|
#in_filename_params = paste0(tolower(gene), "_all_params.csv")
|
||||||
in_filename_params = paste0(tolower(gene), "_comb_stab_struc_params.csv") # part combined for gid
|
in_filename_params = paste0(tolower(gene), "_comb_stab_struc_params.csv") # part combined for gid
|
||||||
infile = paste0(outdir, "/", in_filename_params)
|
infile_params = paste0(outdir, "/", in_filename_params)
|
||||||
cat("\nInput file not specified, assuming filename: ", infile, "\n")
|
cat("\nInput file not specified, assuming filename: ", infile_params, "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
# Get the DFs out of plotting_data()
|
# Get the DFs out of plotting_data()
|
||||||
pd_df = plotting_data(infile)
|
pd_df = plotting_data(infile_params)
|
||||||
my_df = pd_df[[1]]
|
my_df = pd_df[[1]]
|
||||||
my_df_u = pd_df[[2]]
|
my_df_u = pd_df[[2]]
|
||||||
my_df_u_lig = pd_df[[3]]
|
my_df_u_lig = pd_df[[3]]
|
||||||
|
|
|
||||||
|
|
@ -1,433 +1,322 @@
|
||||||
#!/usr/bin/env Rscript
|
#!/usr/bin/env Rscript
|
||||||
#########################################################
|
|
||||||
# TASK: To combine struct params and meta data for plotting
|
|
||||||
# Input csv files:
|
|
||||||
# 1) <gene>_all_params.csv
|
|
||||||
# 2) <gene>_meta_data.csv
|
|
||||||
|
|
||||||
# Output:
|
###########################################################
|
||||||
# 1) muts with opposite effects on stability
|
# TASK: To combine mcsm combined file and meta data.
|
||||||
# 2) large combined df including NAs for AF, OR,etc
|
# This script is sourced from other .R scripts for plotting.
|
||||||
|
###########################################################
|
||||||
|
|
||||||
|
# load libraries and functions
|
||||||
|
|
||||||
|
#==========================================================
|
||||||
|
# combining_dfs_plotting():
|
||||||
|
|
||||||
|
# input args
|
||||||
|
|
||||||
|
## df1_mcsm_comb: <gene>_meta_data.csv
|
||||||
|
## df2_gene_metadata: <gene>_all_params.csv
|
||||||
|
## lig_dist_cutoff = 10, cut off distance
|
||||||
|
|
||||||
|
# Output: returns
|
||||||
|
# 1) large combined df including NAs for AF, OR,etc
|
||||||
# Dim: same no. of rows as gene associated meta_data_with_AFandOR
|
# Dim: same no. of rows as gene associated meta_data_with_AFandOR
|
||||||
# 3) small combined df including NAs for AF, OR, etc.
|
# 2) small combined df including NAs for AF, OR, etc.
|
||||||
# Dim: same as mcsm data
|
# Dim: same as mcsm data
|
||||||
# 4) large combined df excluding NAs
|
# 3) large combined df excluding NAs
|
||||||
# Dim: dim(#1) - na_count_df2
|
# Dim: dim(#1) - na_count_df2
|
||||||
# 5) small combined df excluding NAs
|
# 4) small combined df excluding NAs
|
||||||
# Dim: dim(#2) - na_count_df3
|
# Dim: dim(#2) - na_count_df3
|
||||||
# This script is sourced from other .R scripts for plotting
|
# 5) LIGAND large combined df including NAs for AF, OR,etc
|
||||||
#########################################################
|
# Dim: dim()
|
||||||
#=======================================================================
|
# 6) LIGAND small combined df excluding NAs
|
||||||
# working dir and loading libraries
|
# Dim: dim()
|
||||||
getwd()
|
#==========================================================
|
||||||
setwd("~/git/LSHTM_analysis/scripts/plotting/")
|
combining_dfs_plotting <- function( my_df_u
|
||||||
getwd()
|
, gene_metadata
|
||||||
|
, lig_dist_colname = 'ligand_distance'
|
||||||
require("getopt", quietly = TRUE) # cmd parse arguments
|
, lig_dist_cutoff = 10){
|
||||||
|
# #======================================
|
||||||
# load functions
|
# # 1: Read file: <gene>_meta data.csv
|
||||||
source("Header_TT.R")
|
# #======================================
|
||||||
source("../functions/plotting_globals.R")
|
# cat("\nReading meta data file:", df1_mcsm_comb)
|
||||||
source("../functions/plotting_data.R")
|
#
|
||||||
|
# my_df_u <- read.csv(df1_mcsm_comb
|
||||||
#############################################################
|
# , stringsAsFactors = F
|
||||||
# command line args
|
# , header = T)
|
||||||
#********************
|
# cat("\nDim:", dim(my_df_u))
|
||||||
# !!!FUTURE TODO!!!
|
#
|
||||||
# Can pass additional params of output/plot dir by user.
|
# #======================================
|
||||||
# Not strictly required for my workflow since it is optimised
|
# # 2: Read file: <gene>_meta data.csv
|
||||||
# to have a streamlined input/output flow without filename worries.
|
# #======================================
|
||||||
#********************
|
# cat("\nReading meta data file:", df2_gene_metadata)
|
||||||
spec = matrix(c(
|
#
|
||||||
"drug" ,"d", 1, "character",
|
# gene_metadata <- read.csv(df2_gene_metadata
|
||||||
"gene" ,"g", 1, "character",
|
# , stringsAsFactors = F
|
||||||
"data" ,"f", 2, "character"
|
# , header = T)
|
||||||
), byrow = TRUE, ncol = 4)
|
# cat("\nDim:", dim(gene_metadata))
|
||||||
|
#
|
||||||
opt = getopt(spec)
|
# table(gene_metadata$mutation_info)
|
||||||
|
|
||||||
#FIXME: detect if script running from cmd, then set these
|
# counting NAs in AF, OR cols
|
||||||
drug = opt$drug
|
# or_mychisq
|
||||||
gene = opt$gene
|
if (identical(sum(is.na(my_df_u$or_mychisq))
|
||||||
infile = opt$data
|
, sum(is.na(my_df_u$pval_fisher))
|
||||||
|
, sum(is.na(my_df_u$af)))){
|
||||||
# hardcoding when not using cmd
|
cat("\nPASS: NA count match for OR, pvalue and AF\n")
|
||||||
#drug = "streptomycin"
|
na_count = sum(is.na(my_df_u$af))
|
||||||
#gene = "gid"
|
cat("\nNo. of NAs: ", sum(is.na(my_df_u$or_mychisq)))
|
||||||
|
} else{
|
||||||
if(is.null(drug)|is.null(gene)) {
|
cat("\nFAIL: NA count mismatch"
|
||||||
stop("Missing arguments: --drug and --gene must both be specified (case-sensitive)")
|
, "\nNA in OR: ", sum(is.na(my_df_u$or_mychisq))
|
||||||
}
|
, "\nNA in pvalue: ", sum(is.na(my_df_u$pval_fisher))
|
||||||
|
, "\nNA in AF:", sum(is.na(my_df_u$af)))
|
||||||
#########################################################
|
}
|
||||||
# call functions with relevant args
|
|
||||||
#***********************************
|
# or kin
|
||||||
# import_dirs(): returns
|
if (identical(sum(is.na(my_df_u$or_kin))
|
||||||
# datadir
|
, sum(is.na(my_df_u$pwald_kin))
|
||||||
# indir
|
, sum(is.na(my_df_u$af_kin)))){
|
||||||
# outdir
|
cat("\nPASS: NA count match for OR, pvalue and AF\n from Kinship matrix calculations")
|
||||||
# plotdir
|
na_count = sum(is.na(my_df_u$af_kin))
|
||||||
# dr_muts_col
|
cat("\nNo. of NAs: ", sum(is.na(my_df_u$or_kin)))
|
||||||
# other_muts_col
|
} else{
|
||||||
# resistance_col
|
cat("\nFAIL: NA count mismatch"
|
||||||
#***********************************
|
, "\nNA in OR: ", sum(is.na(my_df_u$or_kin))
|
||||||
import_dirs(drug, gene)
|
, "\nNA in pvalue: ", sum(is.na(my_df_u$pwald_kin))
|
||||||
#***********************************
|
, "\nNA in AF:", sum(is.na(my_df_u$af_kin)))
|
||||||
# plotting_data(): returns
|
}
|
||||||
# my_df
|
|
||||||
# my_df_u
|
str(gene_metadata)
|
||||||
# my_df_u_lig
|
|
||||||
# dup_muts
|
###################################################################
|
||||||
#***********************************
|
# combining: PS
|
||||||
#infile = "/home/tanu/git/Data/streptomycin/output/gid_comb_stab_struc_params.csv"
|
###################################################################
|
||||||
|
# sort by position (same as my_df)
|
||||||
if (!exists("infile") && exists("gene")){
|
head(gene_metadata$position)
|
||||||
#if (!is.character(infile) && exists("gene")){
|
gene_metadata = gene_metadata[order(gene_metadata$position),]
|
||||||
#in_filename_params = paste0(tolower(gene), "_all_params.csv")
|
head(gene_metadata$position)
|
||||||
#in_filename_params = paste0(tolower(gene), "_comb_stab_struc_params.csv") # part combined for gid
|
|
||||||
in_filename_params = paste0(tolower(gene), "_comb_afor.csv") # part combined for gid
|
#=========================
|
||||||
infile = paste0(outdir, "/", in_filename_params)
|
# Merge 1: merged_df2
|
||||||
cat("\nInput file not specified, assuming filename: ", infile, "\n")
|
# dfs with NAs in ORs
|
||||||
}
|
#=========================
|
||||||
|
head(my_df_u$mutationinformation)
|
||||||
# Get the DFs out of plotting_data()
|
head(gene_metadata$mutationinformation)
|
||||||
pd_df = plotting_data(infile)
|
|
||||||
my_df = pd_df[[1]]
|
# Find common columns b/w two df
|
||||||
my_df_u = pd_df[[2]]
|
merging_cols = intersect(colnames(my_df_u), colnames(gene_metadata))
|
||||||
my_df_u_lig = pd_df[[3]]
|
|
||||||
dup_muts = pd_df[[4]]
|
cat(paste0("\nMerging dfs with NAs: big df (1-many relationship b/w id & mut)"
|
||||||
|
, "\nNo. of merging cols:", length(merging_cols)
|
||||||
cat(paste0("Directories imported:"
|
, "\nMerging columns identified:"))
|
||||||
, "\ndatadir:" , datadir
|
|
||||||
, "\nindir:" , indir
|
print(merging_cols)
|
||||||
, "\noutdir:" , outdir
|
|
||||||
, "\nplotdir:" , plotdir))
|
# using all common cols create confusion, so pick one!
|
||||||
|
# merging_cols = merging_cols[[1]]
|
||||||
cat(paste0("\nVariables imported:"
|
merging_cols = 'mutationinformation'
|
||||||
, "\ndrug:" , drug
|
|
||||||
, "\ngene:" , gene
|
cat("\nLinking column being used: mutationinformation")
|
||||||
, "\ngene match:" , gene_match
|
|
||||||
, "\n"))
|
# important checks!
|
||||||
#========================================================
|
table(nchar(my_df_u$mutationinformation))
|
||||||
#===========
|
table(nchar(my_df_u$wild_type))
|
||||||
# input
|
table(nchar(my_df_u$mutant_type))
|
||||||
#===========
|
table(nchar(my_df_u$position))
|
||||||
#in_file1: output of plotting_data.R
|
|
||||||
# my_df_u
|
# all.y because x might contain non-structural positions!
|
||||||
|
merged_df2 = merge(x = gene_metadata
|
||||||
# infile 2: gene associated meta data
|
, y = my_df_u
|
||||||
#in_filename_gene_metadata = paste0(tolower(gene), "_meta_data_with_AFandOR.csv")
|
, by = merging_cols
|
||||||
in_filename_gene_metadata = paste0(tolower(gene), "_metadata.csv")
|
, all.y = T)
|
||||||
infile_gene_metadata = paste0(outdir, "/", in_filename_gene_metadata)
|
|
||||||
cat(paste0("Input infile 2:", infile_gene_metadata))
|
cat("\nDim of merged_df2: ", dim(merged_df2))
|
||||||
|
|
||||||
#===========
|
# Remove duplicate columns
|
||||||
# output
|
dup_cols = names(merged_df2)[grepl("\\.x$|\\.y$", names(merged_df2))]
|
||||||
#===========
|
cat("\nNo. of duplicate cols:", length(dup_cols))
|
||||||
# other variables that you can write
|
check_df_cols = merged_df2[dup_cols]
|
||||||
# primarily called by other scripts for plotting
|
|
||||||
|
identical(check_df_cols$wild_type.x, check_df_cols$wild_type.y)
|
||||||
# PS combined:
|
identical(check_df_cols$position.x, check_df_cols$position.y)
|
||||||
# 1) merged_df2
|
identical(check_df_cols$mutant_type.x, check_df_cols$mutant_type.y)
|
||||||
# 2) merged_df2_comp
|
# False: because some of the ones with OR don't have mutation
|
||||||
# 3) merged_df3
|
identical(check_df_cols$mutation.x, check_df_cols$mutation.y)
|
||||||
# 4) merged_df3_comp
|
|
||||||
|
cols_to_drop = names(merged_df2)[grepl("\\.y",names(merged_df2))]
|
||||||
# LIG combined:
|
cat("\nNo. of cols to drop:", length(cols_to_drop))
|
||||||
# 5) merged_df2_lig
|
|
||||||
# 6) merged_df2_comp_lig
|
# Drop duplicate columns
|
||||||
# 7) merged_df3_lig
|
merged_df2 = merged_df2[,!(names(merged_df2)%in%cols_to_drop)]
|
||||||
# 8) merged_df3_comp_lig
|
|
||||||
|
# Drop the '.x' suffix in the colnames
|
||||||
#%%===============================================================
|
names(merged_df2)[grepl("\\.x$|\\.y$", names(merged_df2))]
|
||||||
|
colnames(merged_df2) <- gsub("\\.x$", "", colnames(merged_df2))
|
||||||
###########################
|
names(merged_df2)[grepl("\\.x$|\\.y$", names(merged_df2))]
|
||||||
# 2: Read file: <gene>_meta data.csv
|
|
||||||
###########################
|
head(merged_df2$position)
|
||||||
cat("Reading meta data file:", infile_gene_metadata)
|
|
||||||
|
# sanity check
|
||||||
gene_metadata <- read.csv(infile_gene_metadata
|
cat("\nChecking nrows in merged_df2")
|
||||||
, stringsAsFactors = F
|
if(nrow(gene_metadata) == nrow(merged_df2)){
|
||||||
, header = T)
|
cat("\nPASS: nrow(merged_df2) = nrow (gene associated gene_metadata)"
|
||||||
cat("Dim:", dim(gene_metadata))
|
,"\nExpected no. of rows: ",nrow(gene_metadata)
|
||||||
|
,"\nGot no. of rows: ", nrow(merged_df2))
|
||||||
table(gene_metadata$mutation_info)
|
} else{
|
||||||
|
cat("\nFAIL: nrow(merged_df2)!= nrow(gene associated gene_metadata)"
|
||||||
# counting NAs in AF, OR cols
|
, "\nExpected no. of rows after merge: ", nrow(gene_metadata)
|
||||||
# or_mychisq
|
, "\nGot no. of rows: ", nrow(merged_df2)
|
||||||
if (identical(sum(is.na(my_df_u$or_mychisq))
|
, "\nFinding discrepancy")
|
||||||
, sum(is.na(my_df_u$pval_fisher))
|
merged_muts_u = unique(merged_df2$mutationinformation)
|
||||||
, sum(is.na(my_df_u$af)))){
|
meta_muts_u = unique(gene_metadata$mutationinformation)
|
||||||
cat("\nPASS: NA count match for OR, pvalue and AF\n")
|
# find the index where it differs
|
||||||
na_count = sum(is.na(my_df_u$af))
|
unique(meta_muts_u[! meta_muts_u %in% merged_muts_u])
|
||||||
cat("\nNo. of NAs: ", sum(is.na(my_df_u$or_mychisq)))
|
quit()
|
||||||
} else{
|
}
|
||||||
cat("\nFAIL: NA count mismatch"
|
|
||||||
, "\nNA in OR: ", sum(is.na(my_df_u$or_mychisq))
|
#=================================================================
|
||||||
, "\nNA in pvalue: ", sum(is.na(my_df_u$pval_fisher))
|
# Merge 2: merged_df3
|
||||||
, "\nNA in AF:", sum(is.na(my_df_u$af)))
|
# dfs with NAs in ORs
|
||||||
}
|
#
|
||||||
|
# Cannot trust lineage, country from this df as the same mutation
|
||||||
# or kin
|
# can have many different lineages
|
||||||
if (identical(sum(is.na(my_df_u$or_kin))
|
# but this should be good for the numerical corr plots
|
||||||
, sum(is.na(my_df_u$pwald_kin))
|
#==================================================================
|
||||||
, sum(is.na(my_df_u$af_kin)))){
|
# remove duplicated mutations
|
||||||
cat("\nPASS: NA count match for OR, pvalue and AF\n from Kinship matrix calculations")
|
cat("\nMerging dfs without NAs: small df (removing muts with no AF|OR associated)"
|
||||||
na_count = sum(is.na(my_df_u$af_kin))
|
,"\nCannot trust lineage info from this"
|
||||||
cat("\nNo. of NAs: ", sum(is.na(my_df_u$or_kin)))
|
,"\nlinking col: mutationinforamtion"
|
||||||
} else{
|
,"\nfilename: merged_df3")
|
||||||
cat("\nFAIL: NA count mismatch"
|
|
||||||
, "\nNA in OR: ", sum(is.na(my_df_u$or_kin))
|
merged_df3 = merged_df2[!duplicated(merged_df2$mutationinformation),]
|
||||||
, "\nNA in pvalue: ", sum(is.na(my_df_u$pwald_kin))
|
head(merged_df3$position); tail(merged_df3$position) # should be sorted
|
||||||
, "\nNA in AF:", sum(is.na(my_df_u$af_kin)))
|
|
||||||
}
|
# sanity check
|
||||||
|
cat("\nChecking nrows in merged_df3")
|
||||||
str(gene_metadata)
|
if(nrow(my_df_u) == nrow(merged_df3)){
|
||||||
|
cat("\nPASS: No. of rows match with my_df"
|
||||||
###################################################################
|
,"\nExpected no. of rows: ", nrow(my_df_u)
|
||||||
# combining: PS
|
,"\nGot no. of rows: ", nrow(merged_df3))
|
||||||
###################################################################
|
} else {
|
||||||
# sort by position (same as my_df)
|
cat("\nFAIL: No. of rows mismatch"
|
||||||
head(gene_metadata$position)
|
, "\nNo. of rows my_df: ", nrow(my_df_u)
|
||||||
gene_metadata = gene_metadata[order(gene_metadata$position),]
|
, "\nNo. of rows merged_df3: ", nrow(merged_df3))
|
||||||
head(gene_metadata$position)
|
quit()
|
||||||
|
}
|
||||||
#=========================
|
|
||||||
# Merge 1: merged_df2
|
# counting NAs in AF, OR cols in merged_df3
|
||||||
# dfs with NAs in ORs
|
# this is because mcsm has no AF, OR cols,
|
||||||
#=========================
|
# so you cannot count NAs
|
||||||
head(my_df_u$mutationinformation)
|
if (identical(sum(is.na(merged_df3$or_kin))
|
||||||
head(gene_metadata$mutationinformation)
|
, sum(is.na(merged_df3$pwald_kin))
|
||||||
|
, sum(is.na(merged_df3$af_kin)))){
|
||||||
# Find common columns b/w two df
|
cat("\nPASS: NA count match for OR, pvalue and AF\n")
|
||||||
merging_cols = intersect(colnames(my_df_u), colnames(gene_metadata))
|
na_count_df3 = sum(is.na(merged_df3$af_kin))
|
||||||
|
cat("\nNo. of NAs: ", sum(is.na(merged_df3$or_kin)))
|
||||||
cat(paste0("Merging dfs with NAs: big df (1-many relationship b/w id & mut)"
|
} else{
|
||||||
, "\nNo. of merging cols:", length(merging_cols)
|
cat("\nFAIL: NA count mismatch"
|
||||||
, "\nMerging columns identified:"))
|
, "\nNA in OR: ", sum(is.na(merged_df3$or_kin))
|
||||||
print(merging_cols)
|
, "\nNA in pvalue: ", sum(is.na(merged_df3$pwald_kin))
|
||||||
|
, "\nNA in AF:", sum(is.na(merged_df3$af_kin)))
|
||||||
# using all common cols create confusion, so pick one!
|
}
|
||||||
# merging_cols = merging_cols[[1]]
|
|
||||||
merging_cols = 'mutationinformation'
|
|
||||||
|
#===================================================
|
||||||
# important checks!
|
# Merge3: merged_df2_comp
|
||||||
table(nchar(my_df_u$mutationinformation))
|
# same as merge 1 but excluding NAs from ORs, etc.
|
||||||
table(nchar(my_df_u$wild_type))
|
#====================================================
|
||||||
table(nchar(my_df_u$mutant_type))
|
cat("\nMerging dfs without any NAs: big df (1-many relationship b/w id & mut)"
|
||||||
table(nchar(my_df_u$position))
|
,"\nfilename: merged_df2_comp")
|
||||||
|
|
||||||
# all.y because x might contain non-structural positions!
|
na_count_df2 = sum(is.na(merged_df2$af))
|
||||||
merged_df2 = merge(x = gene_metadata
|
merged_df2_comp = merged_df2[!is.na(merged_df2$af),]
|
||||||
, y = my_df_u
|
|
||||||
, by = merging_cols
|
# sanity check: no +-1 gymnastics
|
||||||
, all.y = T)
|
cat("\nChecking nrows in merged_df2_comp")
|
||||||
|
if(nrow(merged_df2_comp) == (nrow(merged_df2) - na_count_df2)){
|
||||||
cat("Dim of merged_df2: ", dim(merged_df2))
|
cat("\nPASS: No. of rows match"
|
||||||
|
,"\nDim of merged_df2_comp: "
|
||||||
dup_cols = names(merged_df2)[grepl("\\.x$|\\.y$", names(merged_df2))]
|
,"\nExpected no. of rows: ", nrow(merged_df2) - na_count_df2
|
||||||
cat("\nNo. of duplicate cols:", length(dup_cols))
|
, "\nNo. of rows: ", nrow(merged_df2_comp)
|
||||||
check_df_cols = merged_df2[dup_cols]
|
, "\nNo. of cols: ", ncol(merged_df2_comp))
|
||||||
|
}else{
|
||||||
identical(check_df_cols$wild_type.x, check_df_cols$wild_type.y)
|
cat("\nFAIL: No. of rows mismatch"
|
||||||
identical(check_df_cols$position.x, check_df_cols$position.y)
|
,"\nExpected no. of rows: ", nrow(merged_df2) - na_count_df2
|
||||||
identical(check_df_cols$mutant_type.x, check_df_cols$mutant_type.y)
|
,"\nGot no. of rows: ", nrow(merged_df2_comp))
|
||||||
# False: because some of the ones with OR don't have mutation
|
}
|
||||||
identical(check_df_cols$mutation.x, check_df_cols$mutation.y)
|
|
||||||
|
#======================================================
|
||||||
cols_to_drop = names(merged_df2)[grepl("\\.y",names(merged_df2))]
|
# Merge4: merged_df3_comp
|
||||||
cat("\nNo. of cols to drop:", length(cols_to_drop))
|
# same as merge 2 but excluding NAs from ORs, etc or
|
||||||
|
# remove duplicate mutation information
|
||||||
# subset
|
#=======================================================
|
||||||
merged_df2 = merged_df2[,!(names(merged_df2)%in%cols_to_drop)]
|
na_count_df3 = sum(is.na(merged_df3$af))
|
||||||
|
#merged_df3_comp = merged_df3_comp[!duplicated(merged_df3_comp$mutationinformation),] # a way
|
||||||
# rename the cols with '.x' suffix
|
|
||||||
names(merged_df2)[grepl("\\.x$|\\.y$", names(merged_df2))]
|
merged_df3_comp = merged_df3[!is.na(merged_df3$af),] # another way
|
||||||
colnames(merged_df2) <- gsub("\\.x$", "", colnames(merged_df2))
|
cat("\nChecking nrows in merged_df3_comp")
|
||||||
names(merged_df2)[grepl("\\.x$|\\.y$", names(merged_df2))]
|
|
||||||
|
if(nrow(merged_df3_comp) == (nrow(merged_df3) - na_count_df3)){
|
||||||
#======================================================
|
cat("\nPASS: No. of rows match"
|
||||||
#-------------
|
,"\nDim of merged_df3_comp: "
|
||||||
# DEBUG
|
,"\nExpected no. of rows: ", nrow(merged_df3) - na_count_df3
|
||||||
#-------------
|
, "\nNo. of rows: ", nrow(merged_df3_comp)
|
||||||
merged_df2_g = merged_df2[,!(names(merged_df2)%in%cols_to_drop)]
|
, "\nNo. of cols: ", ncol(merged_df3_comp))
|
||||||
|
}else{
|
||||||
check_cols = colnames(merged_df2)[!colnames(merged_df2)%in%colnames(merged_df2_g)]
|
cat("\nFAIL: No. of rows mismatch"
|
||||||
if ( identical(check_cols, cols_to_drop) ){
|
,"\nExpected no. of rows: ", nrow(merged_df3) - na_count_df3
|
||||||
cat("\nPASS: cols identified have been successfully dropped"
|
,"\nGot no. of rows: ", nrow(merged_df3_comp))
|
||||||
, "\nNo. of cols dropped: ", length(check_cols)
|
}
|
||||||
, "\nNo. of cols in original df: ", ncol(merged_df2)
|
|
||||||
, "\nNo. of cols in revised df: " , ncol(merged_df2_g))
|
# alternate way of deriving merged_df3_comp
|
||||||
}
|
foo = merged_df3[!is.na(merged_df3$af),]
|
||||||
|
bar = merged_df3_comp[!duplicated(merged_df3_comp$mutationinformation),]
|
||||||
#======================================================
|
# compare dfs: foo and merged_df3_com
|
||||||
head(merged_df2$position)
|
all.equal(foo, bar)
|
||||||
|
#summary(comparedf(foo, bar))
|
||||||
# sanity check
|
|
||||||
cat("Checking nrows in merged_df2")
|
#####################################################################
|
||||||
if(nrow(gene_metadata) == nrow(merged_df2)){
|
# Combining: LIG
|
||||||
cat("PASS: nrow(merged_df2) = nrow (gene associated gene_metadata)"
|
#####################################################################
|
||||||
,"\nExpected no. of rows: ",nrow(gene_metadata)
|
|
||||||
,"\nGot no. of rows: ", nrow(merged_df2))
|
#============
|
||||||
} else{
|
# Merges 5-8
|
||||||
cat("FAIL: nrow(merged_df2)!= nrow(gene associated gene_metadata)"
|
#============
|
||||||
, "\nExpected no. of rows after merge: ", nrow(gene_metadata)
|
df_lig = my_df_u[my_df_u[[lig_dist_colname]]<lig_dist_cutoff,]
|
||||||
, "\nGot no. of rows: ", nrow(merged_df2)
|
|
||||||
, "\nFinding discrepancy")
|
merged_df2_lig = merged_df2[merged_df2$ligand_distance<lig_dist_cutoff,]
|
||||||
merged_muts_u = unique(merged_df2$mutationinformation)
|
merged_df2_comp_lig = merged_df2_comp[merged_df2_comp$ligand_distance<lig_dist_cutoff,]
|
||||||
meta_muts_u = unique(gene_metadata$mutationinformation)
|
|
||||||
# find the index where it differs
|
merged_df3_lig = merged_df3[merged_df3$ligand_distance<lig_dist_cutoff,]
|
||||||
unique(meta_muts_u[! meta_muts_u %in% merged_muts_u])
|
merged_df3_comp_lig = merged_df3_comp[merged_df3_comp$ligand_distance<lig_dist_cutoff,]
|
||||||
quit()
|
|
||||||
}
|
# sanity check
|
||||||
|
if (nrow(merged_df3_lig) == nrow(df_lig)){
|
||||||
#=========================
|
print("\nPASS: verified merged_df3_lig")
|
||||||
# Merge 2: merged_df3
|
}else{
|
||||||
# dfs with NAs in ORs
|
cat(paste0("\nFAIL: nrow mismatch for merged_df3_lig"
|
||||||
#
|
, "\nExpected:", nrow(df_lig)
|
||||||
# Cannot trust lineage, country from this df as the same mutation
|
, "\nGot:", nrow(merged_df3_lig)))
|
||||||
# can have many different lineages
|
}
|
||||||
# but this should be good for the numerical corr plots
|
|
||||||
#=========================
|
#==============================================================
|
||||||
# remove duplicated mutations
|
|
||||||
cat("Merging dfs without NAs: small df (removing muts with no AF|OR associated)"
|
############################################
|
||||||
,"\nCannot trust lineage info from this"
|
# OPTIONAL: write output files in one go
|
||||||
,"\nlinking col: mutationinforamtion"
|
############################################
|
||||||
,"\nfilename: merged_df3")
|
#outvars = c(#"merged_df2",
|
||||||
|
#"merged_df2_comp",
|
||||||
merged_df3 = merged_df2[!duplicated(merged_df2$mutationinformation),]
|
#"merged_df2_lig",
|
||||||
head(merged_df3$position); tail(merged_df3$position) # should be sorted
|
#"merged_df2_comp_lig",
|
||||||
|
|
||||||
# sanity check
|
#"meregd_df3_comp"
|
||||||
cat("Checking nrows in merged_df3")
|
#"merged_df3_comp_lig",
|
||||||
if(nrow(my_df_u) == nrow(merged_df3)){
|
#"merged_df3",
|
||||||
cat("PASS: No. of rows match with my_df"
|
#"merged_df3_lig")
|
||||||
,"\nExpected no. of rows: ", nrow(my_df_u)
|
|
||||||
,"\nGot no. of rows: ", nrow(merged_df3))
|
#cat("Writing output files: "
|
||||||
} else {
|
#, "\nPath:", outdir)
|
||||||
cat("FAIL: No. of rows mismatch"
|
|
||||||
, "\nNo. of rows my_df: ", nrow(my_df_u)
|
#for (i in outvars){
|
||||||
, "\nNo. of rows merged_df3: ", nrow(merged_df3))
|
|
||||||
quit()
|
|
||||||
}
|
|
||||||
|
|
||||||
# counting NAs in AF, OR cols in merged_df3
|
|
||||||
# this is because mcsm has no AF, OR cols,
|
|
||||||
# so you cannot count NAs
|
|
||||||
if (identical(sum(is.na(merged_df3$or_kin))
|
|
||||||
, sum(is.na(merged_df3$pwald_kin))
|
|
||||||
, sum(is.na(merged_df3$af_kin)))){
|
|
||||||
cat("PASS: NA count match for OR, pvalue and AF\n")
|
|
||||||
na_count_df3 = sum(is.na(merged_df3$af_kin))
|
|
||||||
cat("No. of NAs: ", sum(is.na(merged_df3$or_kin)))
|
|
||||||
} else{
|
|
||||||
cat("FAIL: NA count mismatch"
|
|
||||||
, "\nNA in OR: ", sum(is.na(merged_df3$or_kin))
|
|
||||||
, "\nNA in pvalue: ", sum(is.na(merged_df3$pwald_kin))
|
|
||||||
, "\nNA in AF:", sum(is.na(merged_df3$af_kin)))
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
#=========================
|
|
||||||
# Merge3: merged_df2_comp
|
|
||||||
# same as merge 1 but excluding NAs from ORs, etc.
|
|
||||||
#=========================
|
|
||||||
cat("Merging dfs without any NAs: big df (1-many relationship b/w id & mut)"
|
|
||||||
,"\nlinking col: Mutationinforamtion"
|
|
||||||
,"\nfilename: merged_df2_comp")
|
|
||||||
|
|
||||||
na_count_df2 = sum(is.na(merged_df2$af))
|
|
||||||
merged_df2_comp = merged_df2[!is.na(merged_df2$af),]
|
|
||||||
|
|
||||||
# sanity check: no +-1 gymnastics
|
|
||||||
cat("Checking nrows in merged_df2_comp")
|
|
||||||
if(nrow(merged_df2_comp) == (nrow(merged_df2) - na_count_df2)){
|
|
||||||
cat("\nPASS: No. of rows match"
|
|
||||||
,"\nDim of merged_df2_comp: "
|
|
||||||
,"\nExpected no. of rows: ", nrow(merged_df2) - na_count_df2
|
|
||||||
, "\nNo. of rows: ", nrow(merged_df2_comp)
|
|
||||||
, "\nNo. of cols: ", ncol(merged_df2_comp))
|
|
||||||
}else{
|
|
||||||
cat("FAIL: No. of rows mismatch"
|
|
||||||
,"\nExpected no. of rows: ", nrow(merged_df2) - na_count_df2
|
|
||||||
,"\nGot no. of rows: ", nrow(merged_df2_comp))
|
|
||||||
}
|
|
||||||
|
|
||||||
#=========================
|
|
||||||
# Merge4: merged_df3_comp
|
|
||||||
# same as merge 2 but excluding NAs from ORs, etc or
|
|
||||||
# remove duplicate mutation information
|
|
||||||
#=========================
|
|
||||||
na_count_df3 = sum(is.na(merged_df3$af))
|
|
||||||
#merged_df3_comp = merged_df3_comp[!duplicated(merged_df3_comp$mutationinformation),] # a way
|
|
||||||
|
|
||||||
merged_df3_comp = merged_df3[!is.na(merged_df3$af),] # another way
|
|
||||||
cat("Checking nrows in merged_df3_comp")
|
|
||||||
|
|
||||||
if(nrow(merged_df3_comp) == (nrow(merged_df3) - na_count_df3)){
|
|
||||||
cat("\nPASS: No. of rows match"
|
|
||||||
,"\nDim of merged_df3_comp: "
|
|
||||||
,"\nExpected no. of rows: ", nrow(merged_df3) - na_count_df3
|
|
||||||
, "\nNo. of rows: ", nrow(merged_df3_comp)
|
|
||||||
, "\nNo. of cols: ", ncol(merged_df3_comp))
|
|
||||||
}else{
|
|
||||||
cat("FAIL: No. of rows mismatch"
|
|
||||||
,"\nExpected no. of rows: ", nrow(merged_df3) - na_count_df3
|
|
||||||
,"\nGot no. of rows: ", nrow(merged_df3_comp))
|
|
||||||
}
|
|
||||||
|
|
||||||
# alternate way of deriving merged_df3_comp
|
|
||||||
foo = merged_df3[!is.na(merged_df3$af),]
|
|
||||||
bar = merged_df3_comp[!duplicated(merged_df3_comp$mutationinformation),]
|
|
||||||
# compare dfs: foo and merged_df3_com
|
|
||||||
all.equal(foo, bar)
|
|
||||||
#summary(comparedf(foo, bar))
|
|
||||||
|
|
||||||
#==============================================================
|
|
||||||
|
|
||||||
#####################################################################
|
|
||||||
# Combining: LIG
|
|
||||||
#####################################################################
|
|
||||||
|
|
||||||
#=========================
|
|
||||||
# Merges 5-8
|
|
||||||
#=========================
|
|
||||||
|
|
||||||
merged_df2_lig = merged_df2[merged_df2$ligand_distance<10,]
|
|
||||||
merged_df2_comp_lig = merged_df2_comp[merged_df2_comp$ligand_distance<10,]
|
|
||||||
|
|
||||||
merged_df3_lig = merged_df3[merged_df3$ligand_distance<10,]
|
|
||||||
merged_df3_comp_lig = merged_df3_comp[merged_df3_comp$ligand_distance<10,]
|
|
||||||
|
|
||||||
# sanity check
|
|
||||||
if (nrow(merged_df3_lig) == nrow(my_df_u_lig)){
|
|
||||||
print("PASS: verified merged_df3_lig")
|
|
||||||
}else{
|
|
||||||
cat(paste0("FAIL: nrow mismatch for merged_df3_lig"
|
|
||||||
, "\nExpected:", nrow(my_df_u_lig)
|
|
||||||
, "\nGot:", nrow(merged_df3_lig)))
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
#==============================================================
|
|
||||||
|
|
||||||
#################
|
|
||||||
# OPTIONAL: write output files in one go
|
|
||||||
#################
|
|
||||||
#outvars = c(#"merged_df2",
|
|
||||||
#"merged_df2_comp",
|
|
||||||
#"merged_df2_lig",
|
|
||||||
#"merged_df2_comp_lig",
|
|
||||||
|
|
||||||
#"meregd_df3_comp"
|
|
||||||
#"merged_df3_comp_lig",
|
|
||||||
#"merged_df3",
|
|
||||||
#"merged_df3_lig")
|
|
||||||
|
|
||||||
#cat("Writing output files: "
|
|
||||||
#, "\nPath:", outdir)
|
|
||||||
|
|
||||||
#for (i in outvars){
|
|
||||||
#out_filename = paste0(i, ".csv")
|
#out_filename = paste0(i, ".csv")
|
||||||
#outfile = paste0(outdir, "/", out_filename)
|
#outfile = paste0(outdir, "/", out_filename)
|
||||||
#cat("Writing output file:"
|
#cat("Writing output file:"
|
||||||
|
|
@ -436,13 +325,10 @@ if (nrow(merged_df3_lig) == nrow(my_df_u_lig)){
|
||||||
#cat("Finished writing: ", outfile
|
#cat("Finished writing: ", outfile
|
||||||
# , "\nNo. of rows: ", nrow(get(i))
|
# , "\nNo. of rows: ", nrow(get(i))
|
||||||
# , "\nNo. of cols: ", ncol(get(i)), "\n")
|
# , "\nNo. of cols: ", ncol(get(i)), "\n")
|
||||||
#}
|
#}
|
||||||
|
|
||||||
# clear variables
|
return(list(merged_df2, merged_df3
|
||||||
rm(foo, bar, gene_metadata
|
, merged_df2_comp, merged_df3_comp
|
||||||
, in_filename_params, infile_params, merging_cols
|
, merged_df2_lig, merged_df3_lig))
|
||||||
, in_filename_gene_metadata, infile_gene_metadata)
|
|
||||||
|
}
|
||||||
#==========================================================================
|
|
||||||
# end of script
|
|
||||||
##==========================================================================
|
|
||||||
|
|
@ -1,442 +0,0 @@
|
||||||
getwd()
|
|
||||||
setwd("~/git/LSHTM_analysis/scripts/plotting/")
|
|
||||||
getwd()
|
|
||||||
|
|
||||||
#########################################################
|
|
||||||
# TASK: To combine struct params and meta data for plotting
|
|
||||||
# Input csv files:
|
|
||||||
# 1) <gene>_all_params.csv
|
|
||||||
# 2) <gene>_meta_data.csv
|
|
||||||
|
|
||||||
# Output:
|
|
||||||
# 1) muts with opposite effects on stability
|
|
||||||
# 2) large combined df including NAs for AF, OR,etc
|
|
||||||
# Dim: same no. of rows as gene associated meta_data_with_AFandOR
|
|
||||||
# 3) small combined df including NAs for AF, OR, etc.
|
|
||||||
# Dim: same as mcsm data
|
|
||||||
# 4) large combined df excluding NAs
|
|
||||||
# Dim: dim(#1) - no. of NAs(AF|OR) + 1
|
|
||||||
# 5) small combined df excluding NAs
|
|
||||||
# Dim: dim(#2) - no. of unique NAs - 1
|
|
||||||
# This script is sourced from other .R scripts for plotting
|
|
||||||
#########################################################
|
|
||||||
|
|
||||||
##########################################################
|
|
||||||
# Installing and loading required packages
|
|
||||||
##########################################################
|
|
||||||
source("Header_TT.R")
|
|
||||||
#require(data.table)
|
|
||||||
#require(arsenal)
|
|
||||||
#require(compare)
|
|
||||||
#library(tidyverse)
|
|
||||||
|
|
||||||
|
|
||||||
#%% variable assignment: input and output paths & filenames
|
|
||||||
drug = "pyrazinamide"
|
|
||||||
gene = "pncA"
|
|
||||||
gene_match = paste0(gene,"_p.")
|
|
||||||
cat(gene_match)
|
|
||||||
|
|
||||||
#=============
|
|
||||||
# directories
|
|
||||||
#=============
|
|
||||||
datadir = paste0("~/git/Data")
|
|
||||||
indir = paste0(datadir, "/", drug, "/input")
|
|
||||||
outdir = paste0("~/git/Data", "/", drug, "/output")
|
|
||||||
|
|
||||||
#===========
|
|
||||||
# input
|
|
||||||
#===========
|
|
||||||
#in_filename = "mcsm_complex1_normalised.csv"
|
|
||||||
in_filename_params = paste0(tolower(gene), "_all_params.csv")
|
|
||||||
infile_params = paste0(outdir, "/", in_filename_params)
|
|
||||||
cat(paste0("Input file 1:", infile_params) )
|
|
||||||
|
|
||||||
# infile 2: gene associated meta data
|
|
||||||
#in_filename_gene_metadata = paste0(tolower(gene), "_meta_data_with_AFandOR.csv")
|
|
||||||
in_filename_gene_metadata = paste0(tolower(gene), "_metadata.csv")
|
|
||||||
infile_gene_metadata = paste0(outdir, "/", in_filename_gene_metadata)
|
|
||||||
cat(paste0("Input infile 2:", infile_gene_metadata))
|
|
||||||
|
|
||||||
#===========
|
|
||||||
# output
|
|
||||||
#===========
|
|
||||||
# mutations with opposite effects
|
|
||||||
out_filename_opp_muts = paste0(tolower(gene), "_muts_opp_effects.csv")
|
|
||||||
outfile_opp_muts = paste0(outdir, "/", out_filename_opp_muts)
|
|
||||||
|
|
||||||
|
|
||||||
#%%===============================================================
|
|
||||||
###########################
|
|
||||||
# Read file: struct params
|
|
||||||
###########################
|
|
||||||
cat("Reading struct params including mcsm:"
|
|
||||||
, in_filename_params)
|
|
||||||
|
|
||||||
mcsm_data = read.csv(infile_params
|
|
||||||
#, row.names = 1
|
|
||||||
, stringsAsFactors = F
|
|
||||||
, header = T)
|
|
||||||
|
|
||||||
cat("Input dimensions:", dim(mcsm_data)) #416, 86
|
|
||||||
|
|
||||||
# clear variables
|
|
||||||
rm(in_filename_params, infile_params)
|
|
||||||
|
|
||||||
str(mcsm_data)
|
|
||||||
|
|
||||||
table(mcsm_data$duet_outcome); sum(table(mcsm_data$duet_outcome) )
|
|
||||||
|
|
||||||
# spelling Correction 1: DUET incase American spelling needed!
|
|
||||||
#mcsm_data$duet_outcome[mcsm_data$duet_outcome=="Stabilising"] <- "Stabilizing"
|
|
||||||
#mcsm_data$duet_outcome[mcsm_data$duet_outcome=="Destabilising"] <- "Destabilizing"
|
|
||||||
|
|
||||||
# checks: should be the same as above
|
|
||||||
table(mcsm_data$duet_outcome); sum(table(mcsm_data$duet_outcome) )
|
|
||||||
head(mcsm_data$duet_outcome); tail(mcsm_data$duet_outcome)
|
|
||||||
|
|
||||||
# spelling Correction 2: Ligand incase American spelling needed!
|
|
||||||
table(mcsm_data$ligand_outcome); sum(table(mcsm_data$ligand_outcome) )
|
|
||||||
#mcsm_data$ligand_outcome[mcsm_data$ligand_outcome=="Stabilising"] <- "Stabilizing"
|
|
||||||
#mcsm_data$ligand_outcome[mcsm_data$ligand_outcome=="Destabilising"] <- "Destabilizing"
|
|
||||||
|
|
||||||
# checks: should be the same as above
|
|
||||||
table(mcsm_data$ligand_outcome); sum(table(mcsm_data$ligand_outcome) )
|
|
||||||
head(mcsm_data$ligand_outcome); tail(mcsm_data$ligand_outcome)
|
|
||||||
|
|
||||||
# muts with opposing effects on protomer and ligand stability
|
|
||||||
table(mcsm_data$duet_outcome != mcsm_data$ligand_outcome)
|
|
||||||
changes = mcsm_data[which(mcsm_data$duet_outcome != mcsm_data$ligand_outcome),]
|
|
||||||
|
|
||||||
# sanity check: redundant, but uber cautious!
|
|
||||||
dl_i = which(mcsm_data$duet_outcome != mcsm_data$ligand_outcome)
|
|
||||||
ld_i = which(mcsm_data$ligand_outcome != mcsm_data$duet_outcome)
|
|
||||||
|
|
||||||
cat("Identifying muts with opposite stability effects")
|
|
||||||
if(nrow(changes) == (table(mcsm_data$duet_outcome != mcsm_data$ligand_outcome)[[2]]) & identical(dl_i,ld_i)) {
|
|
||||||
cat("PASS: muts with opposite effects on stability and affinity correctly identified"
|
|
||||||
, "\nNo. of such muts: ", nrow(changes))
|
|
||||||
}else {
|
|
||||||
cat("FAIL: unsuccessful in extracting muts with changed stability effects")
|
|
||||||
}
|
|
||||||
|
|
||||||
#***************************
|
|
||||||
# write file: changed muts
|
|
||||||
write.csv(changes, outfile_opp_muts)
|
|
||||||
|
|
||||||
cat("Finished writing file for muts with opp effects:"
|
|
||||||
, "\nFilename: ", outfile_opp_muts
|
|
||||||
, "\nDim:", dim(changes))
|
|
||||||
|
|
||||||
# clear variables
|
|
||||||
rm(out_filename_opp_muts, outfile_opp_muts)
|
|
||||||
rm(changes, dl_i, ld_i)
|
|
||||||
|
|
||||||
#***************************
|
|
||||||
# count na in each column
|
|
||||||
na_count = sapply(mcsm_data, function(y) sum(length(which(is.na(y))))); na_count
|
|
||||||
|
|
||||||
# sort by mutationinformation
|
|
||||||
##mcsm_data = mcsm_data[order(mcsm_data$mutationinformation),]
|
|
||||||
##head(mcsm_data$mutationinformation)
|
|
||||||
|
|
||||||
df_ncols = ncol(mcsm_data)
|
|
||||||
|
|
||||||
# REMOVE as this is dangerous due to dup muts
|
|
||||||
# get freq count of positions and add to the df
|
|
||||||
#setDT(mcsm_data)[, occurrence := .N, by = .(position)]
|
|
||||||
|
|
||||||
#cat("Added 1 col: position frequency to see which posn has how many muts"
|
|
||||||
# , "\nNo. of cols now", ncol(mcsm_data)
|
|
||||||
# , "\nNo. of cols before: ", df_ncols)
|
|
||||||
|
|
||||||
#pos_count_check = data.frame(mcsm_data$position, mcsm_data$occurrence)
|
|
||||||
|
|
||||||
# check duplicate muts
|
|
||||||
if (length(unique(mcsm_data$mutationinformation)) == length(mcsm_data$mutationinformation)){
|
|
||||||
cat("No duplicate mutations in mcsm data")
|
|
||||||
}else{
|
|
||||||
dup_muts = mcsm_data[duplicated(mcsm_data$mutationinformation),]
|
|
||||||
dup_muts_nu = length(unique(dup_muts$mutationinformation))
|
|
||||||
cat(paste0("CAUTION:", nrow(dup_muts), " Duplicate mutations identified"
|
|
||||||
, "\nOf these, no. of unique mutations are:", dup_muts_nu
|
|
||||||
, "\nExtracting df with unique mutations only"))
|
|
||||||
mcsm_data_u = mcsm_data[!duplicated(mcsm_data$mutationinformation),]
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nrow(mcsm_data_u) == length(unique(mcsm_data$mutationinformation))){
|
|
||||||
cat("Df without duplicate mutations successfully extracted")
|
|
||||||
} else{
|
|
||||||
cat("FAIL: could not extract clean df!")
|
|
||||||
quit()
|
|
||||||
}
|
|
||||||
|
|
||||||
###########################
|
|
||||||
# 2: Read file: <gene>_meta data.csv
|
|
||||||
###########################
|
|
||||||
cat("Reading meta data file:", infile_gene_metadata)
|
|
||||||
|
|
||||||
gene_metadata <- read.csv(infile_gene_metadata
|
|
||||||
, stringsAsFactors = F
|
|
||||||
, header = T)
|
|
||||||
cat("Dim:", dim(gene_metadata))
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
||||||
# FIXME: remove
|
|
||||||
# counting NAs in AF, OR cols:
|
|
||||||
if (identical(sum(is.na(gene_metadata$OR))
|
|
||||||
, sum(is.na(gene_metadata$pvalue))
|
|
||||||
, sum(is.na(gene_metadata$AF)))){
|
|
||||||
cat("PASS: NA count match for OR, pvalue and AF\n")
|
|
||||||
na_count = sum(is.na(gene_metadata$AF))
|
|
||||||
cat("No. of NAs: ", sum(is.na(gene_metadata$OR)))
|
|
||||||
} else{
|
|
||||||
cat("FAIL: NA count mismatch"
|
|
||||||
, "\nNA in OR: ", sum(is.na(gene_metadata$OR))
|
|
||||||
, "\nNA in pvalue: ", sum(is.na(gene_metadata$pvalue))
|
|
||||||
, "\nNA in AF:", sum(is.na(gene_metadata$AF)))
|
|
||||||
}
|
|
||||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
||||||
# clear variables
|
|
||||||
rm(in_filename_gene_metadata, infile_gene_metadata)
|
|
||||||
|
|
||||||
str(gene_metadata)
|
|
||||||
|
|
||||||
# sort by position (same as mcsm_data)
|
|
||||||
# earlier it was mutationinformation
|
|
||||||
#head(gene_metadata$mutationinformation)
|
|
||||||
#gene_metadata = gene_metadata[order(gene_metadata$mutationinformation),]
|
|
||||||
##head(gene_metadata$mutationinformation)
|
|
||||||
|
|
||||||
head(gene_metadata$position)
|
|
||||||
gene_metadata = gene_metadata[order(gene_metadata$position),]
|
|
||||||
head(gene_metadata$position)
|
|
||||||
|
|
||||||
###########################
|
|
||||||
# Merge 1: two dfs with NA
|
|
||||||
# merged_df2
|
|
||||||
###########################
|
|
||||||
head(mcsm_data$mutationinformation)
|
|
||||||
head(gene_metadata$mutationinformation)
|
|
||||||
|
|
||||||
# Find common columns b/w two df
|
|
||||||
merging_cols = intersect(colnames(mcsm_data), colnames(gene_metadata))
|
|
||||||
|
|
||||||
cat(paste0("Merging dfs with NAs: big df (1-many relationship b/w id & mut)"
|
|
||||||
, "\nNo. of merging cols:", length(merging_cols)
|
|
||||||
, "\nMerging columns identified:"))
|
|
||||||
print(merging_cols)
|
|
||||||
|
|
||||||
#=============
|
|
||||||
# merged_df2): gene_metadata + mcsm_data
|
|
||||||
#==============
|
|
||||||
merged_df2 = merge(x = gene_metadata
|
|
||||||
, y = mcsm_data
|
|
||||||
, by = merging_cols
|
|
||||||
, all.y = T)
|
|
||||||
|
|
||||||
cat("Dim of merged_df2: ", dim(merged_df2) #4520, 11
|
|
||||||
)
|
|
||||||
head(merged_df2$position)
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
||||||
# FIXME: count how many unique muts have entries in meta data
|
|
||||||
# sanity check
|
|
||||||
cat("Checking nrows in merged_df2")
|
|
||||||
if(nrow(gene_metadata) == nrow(merged_df2)){
|
|
||||||
cat("nrow(merged_df2) = nrow (gene associated gene_metadata)"
|
|
||||||
,"\nExpected no. of rows: ",nrow(gene_metadata)
|
|
||||||
,"\nGot no. of rows: ", nrow(merged_df2))
|
|
||||||
} else{
|
|
||||||
cat("nrow(merged_df2)!= nrow(gene associated gene_metadata)"
|
|
||||||
, "\nExpected no. of rows after merge: ", nrow(gene_metadata)
|
|
||||||
, "\nGot no. of rows: ", nrow(merged_df2)
|
|
||||||
, "\nFinding discrepancy")
|
|
||||||
merged_muts_u = unique(merged_df2$mutationinformation)
|
|
||||||
meta_muts_u = unique(gene_metadata$mutationinformation)
|
|
||||||
# find the index where it differs
|
|
||||||
unique(meta_muts_u[! meta_muts_u %in% merged_muts_u])
|
|
||||||
}
|
|
||||||
|
|
||||||
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
|
||||||
|
|
||||||
# sort by position
|
|
||||||
head(merged_df2$position)
|
|
||||||
merged_df2 = merged_df2[order(merged_df2$position),]
|
|
||||||
head(merged_df2$position)
|
|
||||||
|
|
||||||
merged_df2v3 = merge(x = gene_metadata
|
|
||||||
, y = mcsm_data
|
|
||||||
, by = merging_cols
|
|
||||||
, all = T)
|
|
||||||
|
|
||||||
merged_df2v2 = merge(x = gene_metadata
|
|
||||||
, y = mcsm_data
|
|
||||||
, by = merging_cols
|
|
||||||
, all.x = T)
|
|
||||||
#!=!=!=!=!=!=!=!
|
|
||||||
# COMMENT: used all.y since position 186 is not part of the struc,
|
|
||||||
# hence doesn"t have a mcsm value
|
|
||||||
# but 186 is associated with mutation
|
|
||||||
#!=!=!=!=!=!=!=!
|
|
||||||
|
|
||||||
# should be False
|
|
||||||
identical(merged_df2, merged_df2v2)
|
|
||||||
table(merged_df2$position%in%merged_df2v2$position)
|
|
||||||
|
|
||||||
rm(merged_df2v2)
|
|
||||||
|
|
||||||
#!!!!!!!!!!! check why these differ
|
|
||||||
|
|
||||||
#########
|
|
||||||
# merge 3b (merged_df3):remove duplicate mutation information
|
|
||||||
#########
|
|
||||||
cat("Merging dfs without NAs: small df (removing muts with no AF|OR associated)"
|
|
||||||
,"\nCannot trust lineage info from this"
|
|
||||||
,"\nlinking col: Mutationinforamtion"
|
|
||||||
,"\nfilename: merged_df3")
|
|
||||||
|
|
||||||
#==#=#=#=#=#=#
|
|
||||||
# Cannot trust lineage, country from this df as the same mutation
|
|
||||||
# can have many different lineages
|
|
||||||
# but this should be good for the numerical corr plots
|
|
||||||
#=#=#=#=#=#=#=
|
|
||||||
merged_df3 = merged_df2[!duplicated(merged_df2$mutationinformation),]
|
|
||||||
head(merged_df3$position); tail(merged_df3$position) # should be sorted
|
|
||||||
|
|
||||||
# sanity check
|
|
||||||
cat("Checking nrows in merged_df3")
|
|
||||||
if(nrow(mcsm_data) == nrow(merged_df3)){
|
|
||||||
cat("PASS: No. of rows match with mcsm_data"
|
|
||||||
,"\nExpected no. of rows: ", nrow(mcsm_data)
|
|
||||||
,"\nGot no. of rows: ", nrow(merged_df3))
|
|
||||||
} else {
|
|
||||||
cat("FAIL: No. of rows mismatch"
|
|
||||||
, "\nNo. of rows mcsm_data: ", nrow(mcsm_data)
|
|
||||||
, "\nNo. of rows merged_df3: ", nrow(merged_df3))
|
|
||||||
}
|
|
||||||
|
|
||||||
# counting NAs in AF, OR cols in merged_df3
|
|
||||||
# this is becuase mcsm has no AF, OR cols,
|
|
||||||
# so you cannot count NAs
|
|
||||||
if (identical(sum(is.na(merged_df3$OR))
|
|
||||||
, sum(is.na(merged_df3$pvalue))
|
|
||||||
, sum(is.na(merged_df3$AF)))){
|
|
||||||
cat("PASS: NA count match for OR, pvalue and AF\n")
|
|
||||||
na_count_df3 = sum(is.na(merged_df3$AF))
|
|
||||||
cat("No. of NAs: ", sum(is.na(merged_df3$OR)))
|
|
||||||
} else{
|
|
||||||
cat("FAIL: NA count mismatch"
|
|
||||||
, "\nNA in OR: ", sum(is.na(merged_df3$OR))
|
|
||||||
, "\nNA in pvalue: ", sum(is.na(merged_df3$pvalue))
|
|
||||||
, "\nNA in AF:", sum(is.na(merged_df3$AF)))
|
|
||||||
}
|
|
||||||
|
|
||||||
###########################
|
|
||||||
# 4: merging two dfs: without NA
|
|
||||||
###########################
|
|
||||||
#########
|
|
||||||
# merge 4a (merged_df2_comp): same as merge 1 but excluding NA
|
|
||||||
#########
|
|
||||||
cat("Merging dfs without any NAs: big df (1-many relationship b/w id & mut)"
|
|
||||||
,"\nlinking col: Mutationinforamtion"
|
|
||||||
,"\nfilename: merged_df2_comp")
|
|
||||||
|
|
||||||
merged_df2_comp = merged_df2[!is.na(merged_df2$AF),]
|
|
||||||
#merged_df2_comp = merged_df2[!duplicated(merged_df2$mutationinformation),]
|
|
||||||
|
|
||||||
# sanity check
|
|
||||||
cat("Checking nrows in merged_df2_comp")
|
|
||||||
if(nrow(merged_df2_comp) == (nrow(merged_df2) - na_count + 1)){
|
|
||||||
cat("PASS: No. of rows match"
|
|
||||||
,"\nDim of merged_df2_comp: "
|
|
||||||
,"\nExpected no. of rows: ", nrow(merged_df2) - na_count + 1
|
|
||||||
, "\nNo. of rows: ", nrow(merged_df2_comp)
|
|
||||||
, "\nNo. of cols: ", ncol(merged_df2_comp))
|
|
||||||
}else{
|
|
||||||
cat("FAIL: No. of rows mismatch"
|
|
||||||
,"\nExpected no. of rows: ", nrow(merged_df2) - na_count + 1
|
|
||||||
,"\nGot no. of rows: ", nrow(merged_df2_comp))
|
|
||||||
}
|
|
||||||
|
|
||||||
#########
|
|
||||||
# merge 4b (merged_df3_comp): remove duplicate mutation information
|
|
||||||
#########
|
|
||||||
merged_df3_comp = merged_df2_comp[!duplicated(merged_df2_comp$mutationinformation),]
|
|
||||||
|
|
||||||
cat("Dim of merged_df3_comp: "
|
|
||||||
, "\nNo. of rows: ", nrow(merged_df3_comp)
|
|
||||||
, "\nNo. of cols: ", ncol(merged_df3_comp))
|
|
||||||
|
|
||||||
# alternate way of deriving merged_df3_comp
|
|
||||||
foo = merged_df3[!is.na(merged_df3$AF),]
|
|
||||||
# compare dfs: foo and merged_df3_com
|
|
||||||
all.equal(foo, merged_df3)
|
|
||||||
|
|
||||||
summary(comparedf(foo, merged_df3))
|
|
||||||
|
|
||||||
# sanity check
|
|
||||||
cat("Checking nrows in merged_df3_comp")
|
|
||||||
if(nrow(merged_df3_comp) == nrow(merged_df3)){
|
|
||||||
cat("NO NAs detected in merged_df3 in AF|OR cols"
|
|
||||||
,"\nNo. of rows are identical: ", nrow(merged_df3))
|
|
||||||
} else{
|
|
||||||
if(nrow(merged_df3_comp) == nrow(merged_df3) - na_count_df3) {
|
|
||||||
cat("PASS: NAs detected in merged_df3 in AF|OR cols"
|
|
||||||
, "\nNo. of NAs: ", na_count_df3
|
|
||||||
, "\nExpected no. of rows in merged_df3_comp: ", nrow(merged_df3) - na_count_df3
|
|
||||||
, "\nGot no. of rows: ", nrow(merged_df3_comp))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#=============== end of combining df
|
|
||||||
#*********************
|
|
||||||
# writing 1 file in the style of a loop: merged_df3
|
|
||||||
# print(output dir)
|
|
||||||
#i = "merged_df3"
|
|
||||||
#out_filename = paste0(i, ".csv")
|
|
||||||
#outfile = paste0(outdir, "/", out_filename)
|
|
||||||
|
|
||||||
#cat("Writing output file: "
|
|
||||||
# ,"\nFilename: ", out_filename
|
|
||||||
# ,"\nPath: ", outdir)
|
|
||||||
|
|
||||||
#template: write.csv(merged_df3, "merged_df3.csv")
|
|
||||||
#write.csv(get(i), outfile, row.names = FALSE)
|
|
||||||
#cat("Finished writing: ", outfile
|
|
||||||
# , "\nNo. of rows: ", nrow(get(i))
|
|
||||||
# , "\nNo. of cols: ", ncol(get(i)))
|
|
||||||
|
|
||||||
#%% write_output files; all 4 files:
|
|
||||||
outvars = c("merged_df2"
|
|
||||||
, "merged_df3"
|
|
||||||
, "merged_df2_comp"
|
|
||||||
, "merged_df3_comp")
|
|
||||||
|
|
||||||
cat("Writing output files: "
|
|
||||||
, "\nPath:", outdir)
|
|
||||||
|
|
||||||
for (i in outvars){
|
|
||||||
# cat(i, "\n")
|
|
||||||
out_filename = paste0(i, ".csv")
|
|
||||||
# cat(out_filename, "\n")
|
|
||||||
# cat("getting value of variable: ", get(i))
|
|
||||||
outfile = paste0(outdir, "/", out_filename)
|
|
||||||
# cat("Full output path: ", outfile, "\n")
|
|
||||||
cat("Writing output file:"
|
|
||||||
,"\nFilename: ", out_filename,"\n")
|
|
||||||
write.csv(get(i), outfile, row.names = FALSE)
|
|
||||||
cat("Finished writing: ", outfile
|
|
||||||
, "\nNo. of rows: ", nrow(get(i))
|
|
||||||
, "\nNo. of cols: ", ncol(get(i)), "\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
# alternate way to replace with implicit loop
|
|
||||||
# FIXME
|
|
||||||
#sapply(outvars, function(x, y) write.csv(get(outvars), paste0(outdir, "/", outvars, ".csv")))
|
|
||||||
#*************************
|
|
||||||
# clear variables
|
|
||||||
rm(mcsm_data, gene_metadata, foo, drug, gene, gene_match, indir, merged_muts_u, meta_muts_u, na_count, df_ncols, outdir)
|
|
||||||
rm(pos_count_check)
|
|
||||||
#============================= end of script
|
|
||||||
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue