#!/bin/bash
## This file provides the submissions for implementing the k-NN algorithm 
# to acquire the refined furfine output and conduct the testing procedures on the refinement



echo "Running k-NN variant Master Script for Testing the k-NN Refined Furfine Algorithm as presented in Rempel (2016), 'Improving Overnight Loan Identification in Payment Systems'";

echo "Authored by Mark Rempel";

PATH=$PWD/bin:$PATH


# 1) Simulate Payments Data
echo "-----";

echo "Simulating Dummy Payments Data";

#Rcall_serial SimulatePaymentsData.R "Inputs/paymentsdata.RData 20200101 150390"
Rcall_serial SimulatePaymentsData.R  "Inputs/alternative_paymentsdata.RData 20190101 51014"

# 2) Apply Furfine Algorithm to Payments Data

echo "-----";
echo "Applying kNN-refined Furfine algorithm";

Matcall_serial FurfineControlScript 'Inputs/paymentsdata.csv' kNNvariantParameters.txt 'FurfineOutput/kNNrefinedFurfineOutputDummyData'
	# for parallel implementation use: bsub -pack kNN_furfine


echo "-----";
echo "Done Identification";

 # ii)  Convert Furfine csv data to RData

echo "furfdat <- read.csv('FurfineOutput/kNNrefinedFurfineOutputDummyData.csv', header=F); names(furfdat) <- c('date', 'time', 'value', 'sender', 'receiver', 'FXrate', 'ONrate', 'FXind','ONind', 'transnum', 'date_r', 'time_r', 'value_r', 'sender_r', 'receiver_r', 'FXrate_r', 'ONrate_r', 'FXind_r', 'ONind_r', 'transnum_r', 'furf_rate', 'mltpl'); save(furfdat, file='FurfineOutput/kNNrefinedFurfineOutputDummyData.RData');" > convertKNNcsvToRData.R

echo "Converting .csv to RData file";

Rcall_serial convertKNNcsvToRData.R


# 3) Resample days of payments from payments data
        # arguments:    i) 'PaymentsData' in RData format (note, dates must
        #               ii) Output Filename base string
        #               iii) Number of complete iterations, CI, where (B^N = CI x number of lend days per resampled sample)
        #                       Technically B^N = CI x (v) x (vi)
        #               iv) Number of days to be treated as a repayment day for each sampled `lend' day 
        #                       B^R = (iv)
        #               v) Number of lend days per list object (recommend fixing this at 1 and varying (vi) for different number of lend days 
        #               vi) Number of list objects (each containing a lend day and the number of repayment days given by (iv)
        #                       Use this parameter to vary the number of lend days to pair
        #               vii) Random number generator seed      
# a) Resample payments data for the testing
echo "----";
echo "Sampling payments data for testing purposes";

Rcall_serial BootstrapSampling.R "Inputs/paymentsdata_rlvts.RData BootData/ResampledPayments 1 5 1 15 15038"

# b) Resample payments data for training of k-NN
        # to ensure distinct days chosen in this sample compared to the bootstraps above, a different seed is set

echo "----";
echo "Sampling payments data for model training";
Rcall_serial BootstrapSampling.R "Inputs/alternative_paymentsdata_rlvts.RData BootData/ResampledPaymentsForTraining 1 1 1 15 51015"


 # ii) Training for k-NN
# 1st arg: payments to generate `faux' loans from 
# 2nd arg: output filename for the model
# 3rd arg: methodTreatDuplicates (must be set to 'Rempel') 
# 4th arg: Parallel indicator (1 if use matlabpool open to conduct training in parallel)
# 5th arg: number of matlab slaves that will be used (set to 0) for serial
	# note that if nMatSlaves = c > 0 then c slaves will be opened on each core that was specified in bsub submission 
# 6th arg: minimum send time = 6 a.m.
# 7th arg: minimum loan size (for Furfine algorithm)
# 8th arg: lower bound for interest rate filter (deviation from target)
# 9th arg: upper bound for interest rate filter (deviation from target)
# 10th arg: refinement ('None', 'EndOfDay', 'IntRound', 'IntRoundExact')
# 11th arg: SyntheticTraininng (=1 for generating faux-loans, = 0 if have own data - note this case not supported)
# 12th arg: ControlSyntheticForParticipants indicator = 1 (Vestigial parameter) 
# 13th arg: useDANN (must be set to 0) - alternatives not supported
# 14th arg: klossWeighted = parameter for cross-validation in determining optimal k, if choiceK specified then doesn't matter
# 15th arg: choiceK = 0 if want K to be determined via cross-validation, specify some positive (odd) integer otherwise to fix K
# 16th arg: Xpaythreshold is minimum size of payments to draw from (as covariates are normalized, does not need to = 1 million)
# 17th arg: secondary duplicate treatment ('Hendry', 'Armantier')
# 18th arg: training suffix string.
# 19th arg: filename to load furfine data from (to obtain the uniquely identified loans)

echo "----";
echo "Applying Training Procedure to resampled payment data";

Rmatlab_call BootstrapModelTrainingScript.R "BootData/ResampledPaymentsForTraining_NumLendDays_1NumReturnDays_1_Numboots_15_1.RData TrainingData/TrainedDummyDataA Rempel 0 1 60000 1000000 0.5 0.5 None 1 1 0 1 15 0 Hendry kNN FurfineOutput/kNNrefinedFurfineOutputDummyData.RData"

#bsub_RMPI+Matlab 3 BootstrapModelTrainingScript.R --args 'BootData/ResampledPaymentsForTraining_NumLendDays_1NumReturnDays_1_Numboots_15_1.RData' 'TrainingData/TrainedDummyDataA' 'Rempel' 0 1 60000 1000000 0.5 0.5 'None' 1 1 0 1 15 0 'Hendry' 'kNN' 'FurfineOutput/kNNrefinedFurfineOutputDummyData.RData'

# 4) Apply Testing Procedure

# 1st arg: payments to generate `faux' loans from 
# 2nd arg: output filename for the model
# 3rd arg: methodTreatDuplicates (must be set to 'Rempel') 
# 4th arg: Parallel indicator (1 if use matlabpool open to conduct training in parallel)
# 5th arg: number of matlab slaves that will be used (set to 0) for serial
        # note that if nMatSlaves = c > 0 then c slaves will be opened on each core that was specified in bsub submission 
# 6th arg: minimum send time = 6 a.m.
# 7th arg: minimum loan size (for Furfine algorithm)
# 8th arg: lower bound for interest rate filter (deviation from target)
# 9th arg: upper bound for interest rate filter (deviation from target)
# 10th arg: refinement ('None', 'EndOfDay', 'IntRound', 'IntRoundExact')
# 11th arg: SyntheticTraininng (=1 for generating faux-loans, = 0 if have own data - note this case not supported)
# 12th arg: ControlSyntheticForParticipants indicator = 1 (Vestigial parameter) 
# 13th arg: useDANN (must be set to 0) - alternatives not supported
# 14th arg: klossWeighted = parameter for cross-validation in determining optimal k, if choiceK specified then doesn't matter
# 15th arg: choiceK = 0 if want K to be determined via cross-validation, specify some positive (odd) integer otherwise to fix K
# 16th arg: Xpaythreshold is minimum size of payments to draw from (as covariates are normalized, does not need to = 1 million)
# 17th arg: secondary duplicate treatment ('Hendry', 'Armantier')
# 18th arg: training prefix string.
# 19th arg: training suffix string
# 20th arg: model training indicator (= 1 if loading training data from file, 0 otherwise)
# 21st arg: payments data file (.mat format)


echo "----";
echo "Applying Testing Procedure to kNN-refined algorithm";



Rmatlab_call RempelBootstrapTest.R "BootData/ResampledPayments_NumLendDays_1NumReturnDays_5_Numboots_15_1.RData RempelPermutationTestOutputDummyData_1A.RData Rempel 0 1 60000 1000000 0.5 0.5 None 0 1 0 1 15 0 Hendry TrainedDummyDataA_ Sample_1.mat 0 Inputs/paymentsdata.mat"

#bsub_RMPI+Matlab 3 RempelBootstrapTest.R --args 'BootData/ResampledPayments_NumLendDays_1NumReturnDays_5_Numboots_15_1.RData' 'RempelPermutationTestOutputDummyData_1A.RData' 'Rempel' 0 1 60000 1000000 0.5 0.5 'None' 0 1 0 1 15 0 'Hendry' 'TrainedDummyDataA_' 'Sample_1.mat' 0 'Inputs/paymentsdata.mat'


# 5) Summarize Results - Generate fdr statistics


echo "----";
echo "Summarizing results of test procedure";

Rcall_serial SummarizeTestResults_Wrapper.R "Results/RempelPermutationTestOutputDummyData_1A.RData BootData/ResampledPayments_NumLendDays_1NumReturnDays_5_Numboots_15_1.RData FurfineOutput/kNNrefinedFurfineOutputDummyData.RData RempelTypeI 1 Inputs/paymentsdata_rlvts.RData";


# 6) Generating row corresponding to that of row 1 Table 3 in paper and analogous figure to Figure 1 (except for refined variant)

echo "----";
echo "Outputing primary summary results";

Rcall_serial OutputKNNTestingResults.R

# 7) Generating Figure 2 (Intraday value plot Unique vs Faux loans)
#


echo "----";
echo "Collecting Faux loan data";


Matcall_cmd RempelTrainingModelScript 'Inputs/paymentsdata.csv' kNNvariantParameters.txt 'FurfineOutput/FauxLoansDummyData' 'FurfineOutput/BaselineFurfineOutputDummyData.mat';

echo "----";
echo "Generating Figure 2";



Rcall_serial FurfineAndKNNanalysis.R "FurfineOutput/BaselineFurfineOutputDummyData.csv FurfineOutput/FauxLoansDummyData.csv ValueTimeDistributionDummyData 500 500 1000 500 30"


exit 0
