#!/bin/bash
## This script executes the testing of spurious matches for the baseline Furfine algorithm
	# I) Generates simulated data for illustration 
	# II) Applies matching algorithm variant
	# III) Conducts resampling exercise
	# IV) Applies matching algorithm to resampled data
	# V) Generates summary results corresponding to key Figures/Tables in paper


echo "Running Master Script for Testing Baseline Furfine Algorithm as presented in Rempel (2016), 'Improving Overnight Loan Identification in Payment Systems'";

echo "Authored by Mark Rempel, 2015";

PATH=$PWD/bin:$PATH

# 1) Simulate Payments Data

Rcall_serial SimulatePaymentsData.R "Inputs/paymentsdata.RData 20200101 150390";
	# Generates dummy payment data for illustration purposes in RData / csv form with name paymentsdata.[extension]
	# May skip this step and replace `paymentsdata.csv' if supplying own payments data		

echo "Payments data simulated";

# 2) Apply Furfine Algorithm to Payments Data
	# objects required loaded for implementation of Furfine algorithm via `FurfineControlScript.m':
	#	i) Nchunks = number of partitions to split the payments data in to lower the computational burden of algorithm application. 
	#		Nchunks recommended to be such that it divides data into chunks of at least 3 weeks in duration
	#	ii) nMatSlaves - parallel option indicating number of workers to initiate within matlab for parallel computing
	#	iii) min_loansize - indicates dollar level to restrict payments prior to the application of the algorithm
	#	iv) min_sentime - indicates the start of day threshold for application of the furfine algorithm (recommended to be set to start of business day)
	#		presumes data has bee pre-processed to not include payments outside of business hours at the end of day
	#	v) bpsl - lower bound in percentage terms of deviation from target for candidate overnight loans
	#	vi) bpsh - upper bound ...
	#	vii) refinement - choice of `None', `EndOfDay', 'ExactInterest',
	#		a) `EndOfDay' imposes minimum time of day at which an overnight loan may be initiated
	#			i.e. only payments post 4pm would be specified via post4pm <- 160000 
	#		b) `ExactInterest' imposes requirement that implied interest rate is rounded
	#	viii) methodTreatDuplicates - `Hendry'
	#	ix)  Par
	#	x) 
	#
	#	Script.m		input payments data	Alg Parameters		Output base string (will add .mat / .csv extension)

Matcall_serial FurfineControlScript 'Inputs/paymentsdata.csv' BaselineParameters.txt 'FurfineOutput/BaselineFurfineOutputDummyData';
	# For more information on these settings see FurfineControlScript.m
	# 	FurfineControlScript is the master script implementing the Furfine algorithm
	#	subroutines are contained within KNN directory
	
	# Output is saved in FurfineOutput folder in .mat / .csv form 
echo "Baseline Furfine Algorithm Completed";

# 2b) Convert output into RData file
echo "furfdat <- read.csv('FurfineOutput/BaselineFurfineOutputDummyData.csv', header=F); names(furfdat) <- c('date', 'time', 'value', 'sender', 'receiver', 'tranche', 'ONrate', 'jumbo','misc', 'transnum', 'date_r', 'time_r', 'value_r', 'sender_r', 'receiver_r', 'tranche_r', 'ONrate_r', 'jumbo_r', 'misc_r', 'transnum_r', 'furf_rate', 'mltpl'); save(furfdat, file='FurfineOutput/BaselineOriginalFurfineOutputDummyData.RData');" > convertBaselineCSVtoRData.R;

#printf "furfdat <- read.csv('FurfineOutput/BaselineFurfineOutputDummyData.csv', header=F); names(furfdat) <- c('date', 'time', 'value', 'sender', 'receiver', 'tranche', 'ONrate', 'jumbo','misc', 'transnum', 'date_r', 'time_r', 'value_r', 'sender_r', 'receiver_r', 'tranche_r', 'ONrate_r', 'jumbo_r', 'misc_r', 'transnum_r', 'furf_rate', 'mltpl'); save(furfdat, file='FurfineOutput/BaselineOriginalFurfineOutputDummyData.RData');" >> convertBaselineCSVtoRData.R

echo "done";

Rcall_serial convertBaselineCSVtoRData.R;


# 3) Resample days of payments from payments data
	# arguments: 	i) 'PaymentsData' in RData format (note, dates must be in R - date format)
	#		ii) Output Filename base string
	#		iii) Number of complete iterations, CI, where (B^N = CI x number of lend days per resampled sample)
	#			Technically B^N = CI x (v) x (vi)
	#		iv) Number of days to be treated as a repayment day for each sampled `lend' day 
	#			B^R = (iv)
	#		v) Number of lend days per list object (recommend fixing this at 1 and varying (vi) for different number of lend days 
	#		vi) Number of list objects (each containing a lend day and the number of repayment days given by (iv)
	#			Use this parameter to vary the number of lend days to pair
	#		vii) Random number generator seed	

Rcall_serial BootstrapSampling.R "Inputs/paymentsdata_rlvts.RData BootData/ResampledPayments 1 5 1 15 150388";

echo "Bootstrap Sampling of payments completed";

# 4) Apply Testing Procedure 

# 1st arg: payments to generate `faux' loans from 
# 2nd arg: output filename for the model
# 3rd arg: methodTreatDuplicates (must be set to 'Rempel') 
# 4th arg: Parallel indicator (1 if use matlabpool open to conduct training in parallel)
# 5th arg: number of matlab slaves that will be used (set to 0) for serial
        # note that if nMatSlaves = c > 0 then c slaves will be opened on each core that was specified in bsub submission 
# 6th arg: minimum send time = 6 a.m.
# 7th arg: minimum loan size (for Furfine algorithm)
# 8th arg: lower bound for interest rate filter (deviation from target)
# 9th arg: upper bound for interest rate filter (deviation from target)
# 10th arg: refinement ('None', 'EndOfDay', 'IntRound', 'IntRoundExact')

Rmatlab_call RempelBootstrapTest.R "BootData/ResampledPayments_NumLendDays_1NumReturnDays_5_Numboots_15_1.RData BaselinePermutationTestOutputDummyDataNumLend15_1.RData Hendry 0 1 60000 1000000 0.5 0.5 None";

	# for parallel execution on LSF (assuming necessary infrastructure in place):
#bsub_RMPI+Matlab 15 RempelBootstrapTest.R --args 'BootData/ResampledPayments_NumLendDays_1NumReturnDays_5_Numboots_15_1.RData' 'BaselinePermutationTestOutputDummyDataNumLend15_1.RData' 'Hendry' 0 1 60000 1000000 0.5 0.5 'None';

echo "Bootstrap Testing procedure on Baseline Algorithm complete";


# 5) Summarize Results - Estimate (positive) False Discovery Rates

Rcall_serial SummarizeTestResults_Wrapper.R "Results/BaselinePermutationTestOutputDummyDataNumLend15_1.RData BootData/ResampledPayments_NumLendDays_1NumReturnDays_5_Numboots_15_1.RData FurfineOutput/BaselineOriginalFurfineOutputDummyData.RData BaselineTypeI 1 Inputs/paymentsdata_rlvts.RData";

echo "Baseline Results summarized";

# 6) Output summary table / figures 

Rcall_serial OutputBaselineTestingResults.R;
# this will output a pdf and table (in .csv) analogous to row 1 of Table 1 and Figure 1 in the paper

echo "Table 1 / Figure 1 results outputted";

# 7) Output of FX summary Figure


Rcall_serial FXexposure.R "FXconfigurationAndSpuriousMatches_SimulatedData 0 Inputs/SimulatedFXandONrateData.RData 192564 1 days";



#Rcall_serial /home/mfa/remm/Replication/FXexposure.R "/home/mfa/remm/Replication/FXconfigurationAndSpuriousMatches Inputs/USDCAN_FX_MinCloseQuotes_2001-2013.RData 0 192564 15 months 2005-01-01 2012-01-01"


echo "Figure 3 - FX exposure Figure plotted. Master replication command script complete.";
exit 0

