#########################################################
# TASK: To combine mcsm and meta data with af and or   
# This script doesn't output anything, but can do if needed.
# This script is sourced from other .R scripts for plotting
#########################################################
getwd()
setwd('~/git/LSHTM_analysis/mcsm_analysis/pyrazinamide/scripts/')
getwd()

##########################################################
# 				Installing and loading required packages 			 
##########################################################

source('Header_TT.R')
#require(data.table)
#require(arsenal)
#require(compare)
#library(tidyverse)

#################################
# Read file: normalised file
# output of step 4 mcsm_pipeline
#################################

#%% variable assignment: input and output paths & filenames
drug = 'pyrazinamide'
gene = 'pncA'
gene_match = paste0(gene,'_p.')
cat(gene_match)

#===========
# input
#===========
# infile1: mCSM data
#indir = '~/git/Data/pyrazinamide/input/processed/'
indir = paste0('~/git/Data', '/', drug, '/', 'output') # revised {TODO: change in mcsm pipeline}
in_filename = 'mcsm_complex1_normalised.csv'
infile = paste0(indir, '/', in_filename)
cat(paste0('Reading infile1: mCSM output file', ' ', infile) )

# infile2: gene associated meta data combined with AF and OR
#indir: same as above
in_filename_comb = paste0(tolower(gene),  '_meta_data_with_AFandOR.csv')
infile_comb = paste0(indir, '/', in_filename_comb)
cat(paste0('Reading infile2: gene associated combined metadata:', infile_comb))

#===========
# output
#===========
# Uncomment if and when required to output
outdir = paste0('~/git/Data', '/', drug, '/', 'output') #same as indir
cat('Output dir: ', outdir)
#out_filename = paste0(tolower(gene), 'XXX')
#outfile = paste0(outdir, '/', out_filename)
#cat(paste0('Output file with full path:', outfile))
#%% end of variable assignment for input and output files

#################################
# Read file: normalised file
# output of step 4 mcsm_pipeline
#################################
cat('Reading mcsm_data:'
    , '\nindir: ', indir
    , '\ninfile_comb: ', in_filename)

mcsm_data = read.csv(infile
                     , row.names = 1
                     , stringsAsFactors = F
                     , header = T)

cat('Read mcsm_data file:'
    , '\nNo.of rows: ', nrow(mcsm_data)
    , '\nNo. of cols:', ncol(mcsm_data))

# clear variables
rm(in_filename, infile)

str(mcsm_data)

table(mcsm_data$DUET_outcome); sum(table(mcsm_data$DUET_outcome) )

# spelling Correction 1: DUET
mcsm_data$DUET_outcome[mcsm_data$DUET_outcome=='Stabilizing'] <- 'Stabilising'
mcsm_data$DUET_outcome[mcsm_data$DUET_outcome=='Destabilizing'] <- 'Destabilising'

# checks: should be the same as above
table(mcsm_data$DUET_outcome); sum(table(mcsm_data$DUET_outcome) )
head(mcsm_data$DUET_outcome); tail(mcsm_data$DUET_outcome)

# spelling Correction 2: Ligand
table(mcsm_data$Lig_outcome); sum(table(mcsm_data$Lig_outcome) )

mcsm_data$Lig_outcome[mcsm_data$Lig_outcome=='Stabilizing'] <- 'Stabilising'
mcsm_data$Lig_outcome[mcsm_data$Lig_outcome=='Destabilizing'] <- 'Destabilising'

# checks: should be the same as above
table(mcsm_data$Lig_outcome); sum(table(mcsm_data$Lig_outcome) )
head(mcsm_data$Lig_outcome); tail(mcsm_data$Lig_outcome)

# muts with opposing effects on protomer and ligand stability
table(mcsm_data$DUET_outcome != mcsm_data$Lig_outcome)
changes = mcsm_data[which(mcsm_data$DUET_outcome != mcsm_data$Lig_outcome),]

# sanity check: redundant, but uber cautious!
dl_i = which(mcsm_data$DUET_outcome != mcsm_data$Lig_outcome)
ld_i = which(mcsm_data$Lig_outcome != mcsm_data$DUET_outcome)

if(nrow(changes) == (table(mcsm_data$DUET_outcome != mcsm_data$Lig_outcome)[[2]]) & identical(dl_i,ld_i)) {
  cat('PASS: muts with opposite effects on stability and affinity identified correctly'
        , '\nNo. of such muts: ', nrow(changes))
}else {
  cat('FAIL: unsuccessful in extracting muts with changed stability effects')
}

#***************************
# write file: changed muts
out_filename = 'muts_opp_effects.csv'
outfile = paste0(outdir, '/', out_filename)
cat('Writing file for muts with opp effects:'
    , '\nFilename: ', outfile
    , '\nPath: ', outdir)

write.csv(changes, outfile)
#****************************
# clear variables
rm(out_filename, outfile)
rm(changes, dl_i, ld_i)

# count na in each column
na_count = sapply(mcsm_data, function(y) sum(length(which(is.na(y))))); na_count

# sort by Mutationinformation
mcsm_data = mcsm_data[order(mcsm_data$Mutationinformation),]
head(mcsm_data$Mutationinformation)

orig_col = ncol(mcsm_data)
# get freq count of positions and add to the df
setDT(mcsm_data)[, occurrence := .N, by = .(Position)] 

cat('Added 1 col: position frequency to see which posn has how many muts'
    , '\nNo. of cols now', ncol(mcsm_data)
    , '\nNo. of cols before: ', orig_col)

pos_count_check = data.frame(mcsm_data$Position, mcsm_data$occurrence)

###########################
# 2: Read file: meta data with AFandOR
###########################
cat('Reading combined meta data and AFandOR file:'
    , '\nindir: ', indir
    , '\ninfile_comb: ', in_filename_comb)

meta_with_afor <- read.csv(infile_comb
                      , stringsAsFactors = F
                      , header = T)

cat('Read mcsm_data file:'
    , '\nNo.of rows: ', nrow(meta_with_afor)
    , '\nNo. of cols:', ncol(meta_with_afor))

# clear variables
rm(in_filename_comb, infile_comb)

str(meta_with_afor)

# sort by Mutationinformation
head(meta_with_afor$Mutationinformation)
meta_with_afor = meta_with_afor[order(meta_with_afor$Mutationinformation),]
head(meta_with_afor$Mutationinformation)

###########################
# 3: merging two dfs: with NA
###########################
# link col name  = 'Mutationinforamtion'
cat('Merging dfs with NAs: big df (1-many relationship b/w id & mut)'
    ,'\nlinking col: Mutationinforamtion'
    ,'\nfilename: merged_df2')

head(mcsm_data$Mutationinformation)
head(meta_with_afor$Mutationinformation)

#########
# merge 3a: meta data with mcsm
#########
merged_df2 = merge(x = meta_with_afor
                  ,y = mcsm_data
                  , by = 'Mutationinformation'
                  , all.y = T)

cat('Dim of merged_df2: '
    , '\nNo. of rows: ', nrow(merged_df2)
    , '\nNo. of cols: ', ncol(merged_df2))
head(merged_df2$Position)

if(nrow(meta_with_afor) == nrow(merged_df2)){
  cat('nrow(merged_df2) = nrow (gene associated metadata)'
      ,'\nExpected no. of rows: ',nrow(meta_with_afor) 
      ,'\nGot no. of rows: ', nrow(merged_df2))
} else{
  cat('nrow(merged_df2)!= nrow(gene associated metadata)'
      , '\nExpected no. of rows after merge: ', nrow(meta_with_afor)
      , '\nGot no. of rows: ', nrow(merged_df2)
      , '\nFinding discrepancy')
  merged_muts_u = unique(merged_df2$Mutationinformation)
  meta_muts_u = unique(meta_with_afor$Mutationinformation)
    # find the index where it differs
  unique(meta_muts_u[! meta_muts_u %in% merged_muts_u])
}

# sort by Position
head(merged_df2$Position)
merged_df2 = merged_df2[order(merged_df2$Position),]
head(merged_df2$Position)

merged_df2v2 = merge(x = meta_with_afor
                   ,y = mcsm_data
                   , by = 'Mutationinformation'
                   , all.x = T) 
#!=!=!=!=!=!=!=!
# COMMENT: used all.y since position 186 is not part of the struc,
# hence doesn't have a mcsm value
# but 186 is associated with mutation
#!=!=!=!=!=!=!=!

# should  be False
identical(merged_df2, merged_df2v2)
table(merged_df2$Position%in%merged_df2v2$Position)

rm(merged_df2v2)

#########
# merge 3b:remove duplicate mutation information
#########
cat('Merging dfs with NAs: small df (removing duplicate muts)'
    ,'\nCannot trust lineage info from this'
    ,'\nlinking col: Mutationinforamtion'
    ,'\nfilename: merged_df3')

#==#=#=#=#=#=#
# Cannot trust lineage, country from this df as the same mutation
# can have many different lineages
# but this should be good for the numerical corr plots
#=#=#=#=#=#=#=
merged_df3 = merged_df2[!duplicated(merged_df2$Mutationinformation),] 
head(merged_df3$Position); tail(merged_df3$Position) # should be sorted

# sanity checks
# nrows of merged_df3 should be the same as the nrows of mcsm_data
if(nrow(mcsm_data) == nrow(merged_df3)){
  cat('PASS: No. of rows match with mcsm_data'
      ,'\nExpected no. of rows: ', nrow(mcsm_data)
      ,'\nGot no. of rows: ', nrow(merged_df3))
} else {
  cat('FAIL: No. of rows mismatch'
      , '\nNo. of rows mcsm_data: ', nrow(mcsm_data)
      , '\nNo. of rows merged_df3: ', nrow(merged_df3))
}

#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# uncomment as necessary
# only need to run this if merged_df2v2 i.e non structural pos included
#mcsm = mcsm_data$Mutationinformation
#my_merged = merged_df3$Mutationinformation

# find the index where it differs
#diff_n = which(!my_merged%in%mcsm)

#check if it is indeed pos 186
#merged_df3[diff_n,]

# remove this entry
#merged_df3 = merged_df3[-diff_n,]]
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

###########################
# 4: merging two dfs: without NA
###########################
cat('Merging dfs without any NAs: big df (1-many relationship b/w id & mut)'
    ,'\nlinking col: Mutationinforamtion'
    ,'\nfilename: merged_df2_comp')

#########
# merge 4a: same as merge 1 but excluding NA
#########
merged_df2_comp = merged_df2[!is.na(merged_df2$AF),] 
#merged_df2_comp = merged_df2[!duplicated(merged_df2$Mutationinformation),] 

cat('Dim of merged_df2_comp: '
    , '\nNo. of rows: ', nrow(merged_df2_comp)
    , '\nNo. of cols: ', ncol(merged_df2_comp))

#########
# merge 4b: remove duplicate mutation information
#########
merged_df3_comp = merged_df2_comp[!duplicated(merged_df2_comp$Mutationinformation),] 

cat('Dim of merged_df3_comp: '
    , '\nNo. of rows: ', nrow(merged_df3_comp)
    , '\nNo. of cols: ', ncol(merged_df3_comp))

# alternate way of deriving merged_df3_comp
foo = merged_df3[!is.na(merged_df3$AF),]
# compare dfs: foo and merged_df3_com
all.equal(foo, merged_df3)

summary(comparedf(foo, merged_df3))

#=============== end of combining df
#*********************
# write_output files
# output dir
#outdir = '~/git/Data/pyrazinamide/output/'
#uncomment as necessary
#FIXME
#out_filenames = c('merged_df2'
#             , 'merged_df3'
#             , 'meregd_df2_comp'
#             , 'merged_df3_comp'
#)

#cat('Writing output files: '
#    , '\nPath:', outdir)

#for (i in out_filenames){
#  print(i)
#  print(get(i))
#  outvar = get(i)
#  print(outvar)
#  outfile = paste0(outdir, '/', outvar, '.csv')
#  cat('Writing output file:'
#      ,'\nFilename: ', outfile
#      ,'\n')
#  write.csv(outvar, outfile)
#  cat('Finished writing file:'
#      ,'\nNo. of rows:', nrow(outvar)
#      , '\nNo. of cols:', ncol(outvar))
#}

#sapply(out_filenames, function(x) write.csv(x, 'x.csv'))
#*************************
# clear variables
rm(mcsm_data, meta_with_afor, foo, drug, gene, gene_match, indir, merged_muts_u, meta_muts_u, na_count, orig_col, outdir)
rm(pos_count_check)
#============================= end of script