#!/bin/sh
# \
exec oagtclsh "$0" "$@"

if ![info exists env(LOCO_BINDIR)] {
    puts stderr "Error: LOCO_BINDIR environment variable is not defined"; exit
}
source $env(LOCO_BINDIR)/locoCommonProcedures

#puts stdout "locoRemoteCalculateDerivative ([exec hostname]): LOCO_BINDIR $env(LOCO_BINDIR)"

set usage "locoRemoteCalculateDerivative -mode <copy|run|cleanup|localRun> <other-options>"
set usageCopy "locoRemoteCalculateDerivative -remotePath <hostname:dirname> -optionsFile <filename> -mode copy -copyFiles <file-list> -doneFile <filename> -abortFile <filename> -errorFile <filename> -matrixFile <filename> \[-waitInterval <sec>\] \[-verbose 1\]"
set usageRun "locoRemoteCalculateDerivative -remotePath <hostname:dirname> -optionsFile <filename> -mode run -doneFile <filename> -errorFile <filename> -matrixFile <filename> \[-verbose <0|1>\]"
set args $argv
set usageLocalRun "locoRemoteCalculateDerivative -workDir -varList -splitTasks -scriptName -scriptParameters -matrixFile -queueDirName -rootTaskName -useQsub -continuePrevious -waitTime -qsubCommand -waitInterval -submissionPause -verbose"

set mode ""
APSParseArguments {mode}
if ![string length $mode] {
    puts stderr $usage
    exit
}

switch -exact $mode {
    copy {
	set requiredArgList [list remotePath optionsFile matrixFile copyFiles doneFile abortFile errorFile]
	set argList [concat $requiredArgList waitInterval verbose]
    }
    run {
	set requiredArgList [list remotePath optionsFile matrixFile doneFile errorFile]
	set argList [concat $requiredArgList waitInterval verbose]
    }
    cleanup {
	set requiredArgList [list remotePath optionsFile copyFiles]
	set argList $requiredArgList
    }
    localRun {
	set requiredArgList [list workDir varList splitTasks scriptName scriptParameters matrixFile queueDirName rootTaskName useQsub continuePrevious waitTime \
				 qsubCommand qsubRespProcCommand queueSystemName submissionPause doneFile errorFile]
	set argList [concat $requiredArgList colPrefix colSuffix waitInterval verbose]
    }
}
foreach arg $argList {set arg ""}
set waitInterval 2
set verbose 0
APSParseArguments $argList
set error 0
foreach arg $requiredArgList {if ![string length [set $arg]] {puts stdout "Argument $arg is missing."; set error 1}}
if $error {puts stderr $usage; exit}


#--------------------------------------------------------------------------------------------------------------------------
proc ReportErrorAndStop {args} {
    APSParseArguments {errorFile stopMessage}
    puts stderr "Error: see $errorFile"
    set fid [open $errorFile w]
    puts $fid $stopMessage
    close $fid
    exit
}
#--------------------------------------------------------------------------------------------------------------------------
proc ChangeColumnNames {args} {
    APSParseArguments {matrixFile colPrefix colSuffix}
    exec sddsconvert $matrixFile -pipe=out "-editName=col,*,a i%$colPrefix% e i%$colSuffix%" \
	| sddsconvert -pipe=in $matrixFile.edit -rename=col,${colPrefix}Rootname${colSuffix}=Rootname
    file copy -force $matrixFile.edit $matrixFile
    file delete $matrixFile.edit
}
#--------------------------------------------------------------------------------------------------------------------------

#catch {file delete $doneFile $abortFile $errorFile}
#set remoteHostName [string range $remotePath 0 [expr [string first : $remotePath] - 1]]
#set remoteWorkDir [string range $remotePath [expr [string first : $remotePath] + 1] [string length $remotePath]]

switch -exact $mode {
    copy {
	#------ This branch is run on the local host, it copies all required files to the remote host and then starts
	#------ itself on the remote host.

	catch {file delete $doneFile $abortFile $errorFile}
	set remoteHostName [string range $remotePath 0 [expr [string first : $remotePath] - 1]]
	set remoteWorkDir [string range $remotePath [expr [string first : $remotePath] + 1] [string length $remotePath]]

	#------ Copy files to remote host
	if [catch {exec ssh $remoteHostName "ls $remoteWorkDir"} result] {
	    after 1000
	    if [catch {exec ssh $remoteHostName "mkdir $remoteWorkDir"} result] {
		ReportErrorAndStop -errorFile $errorFile -stopMessage "Error making remote dir: $result" 
	    }
	}
	set copiedFiles ""
	foreach file $copyFiles {
	    exec scp $file $remoteHostName:$remoteWorkDir/[file tail $file]
	    lappend copiedFiles $remoteWorkDir/[file tail $file]
	}
	lappend copiedFiles $remoteWorkDir/[file tail $matrixFile]
	
	#------ Starting program on the remote host
	set remoteDoneFile $remoteWorkDir/[file tail $doneFile]
	set remoteErrorFile $remoteWorkDir/[file tail $errorFile]
	if $verbose {
	    puts stdout "Running on remote host $remoteHostName:"
	    puts stdout "ssh $remoteHostName \"$remoteWorkDir/locoRemoteCalculateDerivative -remotePath $remotePath \
                         -optionsFile $remoteWorkDir/[file tail $optionsFile] -mode run -doneFile $remoteDoneFile \
                         -matrixFile $remoteWorkDir/[file tail $matrixFile] -errorFile $remoteErrorFile -verbose $verbose\""
	}
#	exec ssh $remoteHostName "setenv LOCO_BINDIR $remoteWorkDir; $remoteWorkDir/locoRemoteCalculateDerivative -remotePath $remotePath \
#            -optionsFile $remoteWorkDir/[file tail $optionsFile] -mode run -doneFile $remoteDoneFile \
#            -matrixFile $remoteWorkDir/[file tail $matrixFile] -errorFile $remoteErrorFile -verbose $verbose" &
	exec ssh $remoteHostName "export LOCO_BINDIR=$remoteWorkDir; $remoteWorkDir/locoRemoteCalculateDerivative -remotePath $remotePath \
            -optionsFile $remoteWorkDir/[file tail $optionsFile] -mode run -doneFile $remoteDoneFile \
            -matrixFile $remoteWorkDir/[file tail $matrixFile] -errorFile $remoteErrorFile -verbose $verbose" &
	
	#------ Waiting for the done-file to appear on remote host then copy the matrix file to local host:
	if $verbose {puts stdout "Waiting for jobs to complete on remote host $remoteHostName..."}
	set done 0
	while  {$done == 0} {
	    after [expr $waitInterval * 1000]
	    if [catch {exec ssh $remoteHostName "ls $remoteDoneFile"} result] {
		#------ Abort if requested...
		if [file exists $abortFile] {
		    #------ If we remove all files from remote host, jobs will die away by themselves
		    #------ Several times to make sure locked files are removed
		    for {set I 0} {$I < 3} {incr I} {
#			catch {exec ssh $remoteHostName "setenv LOCO_BINDIR $remoteWorkDir; $remoteWorkDir/locoRemoteCalculateDerivative \
#                               -remotePath $remotePath -optionsFile $remoteWorkDir/[file tail optionsFile] -mode cleanup -copyFiles \"$copiedFiles\" "}
			catch {exec ssh $remoteHostName "export LOCO_BINDIR=$remoteWorkDir; $remoteWorkDir/locoRemoteCalculateDerivative \
                               -remotePath $remotePath -optionsFile $remoteWorkDir/[file tail optionsFile] -mode cleanup -copyFiles \"$copiedFiles\" "}
			after 1000
		    }
		    exit
		}
		#------ Check for error file...
		if ![catch {exec ssh $remoteHostName "ls $remoteErrorFile"} result] {
		    exec scp $remoteHostName:$remoteErrorFile $errorFile
		    exit
		}
	    } else {
		set done 1
	    }
	}
	
	#------ Copying matrix file from the remote host:
	if $verbose {puts stdout "Copying matrix file from $remoteHostName..."}
	if [catch {exec scp $remoteHostName:$remoteWorkDir/[file tail $matrixFile] $matrixFile} result] {
	    ReportErrorAndStop -errorFile $errorFile -stopMessage "Error copying final file: $result"
	}
	#------ Making local done-file (the file that the program that called remoteCalculateDerivative is waiting for):
	if [catch {open $doneFile w} fid] { ReportErrorAndStop -errorFile $errorFile -stopMessage "$fid" }
	puts $fid "Done."
	close $fid
	
	#------ Cleaning up the remote host directory:
	if $verbose {puts stdout "Cleaning up files on remote host $remoteHostName..."}
	file delete $optionsFile
	lappend copiedFiles $remoteDoneFile
#	catch {exec ssh $remoteHostName "setenv LOCO_BINDIR $remoteWorkDir; $remoteWorkDir/locoRemoteCalculateDerivative -remotePath $remotePath \
#            -optionsFile $remoteWorkDir/[file tail $optionsFile] -mode cleanup \
#	    -copyFiles \"[concat $copiedFiles $remoteDoneFile]\" " &}
	catch {exec ssh $remoteHostName "export LOCO_BINDIR=$remoteWorkDir; $remoteWorkDir/locoRemoteCalculateDerivative -remotePath $remotePath \
            -optionsFile $remoteWorkDir/[file tail $optionsFile] -mode cleanup \
	    -copyFiles \"[concat $copiedFiles $remoteDoneFile]\" " &}
    }
    run {
	#------ This branch is run on the remote host, it calculates RMD.
	catch {file delete $doneFile $errorFile}
	set remoteHostName [string range $remotePath 0 [expr [string first : $remotePath] - 1]]
	set remoteWorkDir [string range $remotePath [expr [string first : $remotePath] + 1] [string length $remotePath]]

	switch -exact $remoteHostName {
	    weed {
		set env(SGE_ROOT) /act/sge
		set env(PATH) $env(PATH):/act/sge/bin/lx24-amd64
	    }
	    apex {
		set env(ELEGANT_BINDIR) /data/bin
	    }
	    orthros {
	    }
	    default {
		ReportErrorAndStop -errorFile $errorFile -stopMessage "Host $remoteHostName is not supported."
	    }
	}
	set env(LOCO_BINDIR) $remoteWorkDir
	set locoBinDir $remoteWorkDir
	set auto_path [linsert $auto_path 0 $locoBinDir]

	#------ workDir required inside initialDefinitions.tcl
	set workDir $remoteWorkDir
	source $optionsFile
	source $workDir/locoCommonProcedures
	if [file exists $workDir/initialDefinitions.tcl] { source $workDir/initialDefinitions.tcl }
	
	if [catch {Fit_CalculateResponseMatrixDerivative -varList $varList \
		       -splitTasks $splitTasks -verbose $verbose \
		       -scriptName $scriptName -scriptParameters $scriptParameters \
		       -matrixFile $matrixFile -tmpDirName $queueDirName -rootTaskName $rootTaskName \
		       -useQsub $useQsub -continuePrevious $continuePrevious -usePopupWindow 0 \
		       -waitTime $waitTime -qsubCommand $qsubCommand \
		       -waitInterval $waitInterval -submissionPause $submissionPause \
		       -abortFile [APSTmpString]} result] {
	    ReportErrorAndStop -errorFile $errorFile -stopMessage "Fit_CalculateResponseMatrixDerivative: $result"
	}
	if [info exists colPrefix] {
	    if [string length $colPrefix] {
		if [catch {exec sddsconvert $matrixFile -pipe=out "-editNames=col,*,a i%${colPrefix}%" \
			       | sddsconvert -pipe=in $matrixFile.tmp -rename=col,${colPrefix}Rootname=Rootname} result] {
		    ReportErrorAndStop -errorFile $errorFile -stopMessage "Error changing column names: $result"
		}
		file copy -force $matrixFile.tmp $matrixFile
		file delete $matrixFile.tmp
	    }
	}
	if [info exists colSuffix] {
	    if [string length $colSuffix] {
		if [catch {exec sddsconvert $matrixFile -pipe=out "-editNames=col,*,e i%${colSuffix}%" \
			       | sddsconvert -pipe=in $matrixFile.tmp1 -rename=col,Rootname$colSuffix=Rootname} result] {
		    ReportErrorAndStop -errorFile $errorFile -stopMessage "Error changing column names: $result"
		}
		file copy -force $matrixFile.tmp1 $matrixFile
		file delete $matrixFile.tmp1
	    }
	}
	if [catch {open $doneFile w} fid] { 
	    ReportErrorAndStop -errorFile $errorFile -stopMessage "$fid"
	} else  {
	    puts $fid "Done."
	    close $fid
	}
    }
    cleanup {
	#------ This branch removes files from the remote host.
	set remoteWorkDir [string range $remotePath [expr [string first : $remotePath] + 1] [string length $remotePath]]
	set workDir $remoteWorkDir
	source $optionsFile
	eval file delete $copyFiles
	for {set I 0} {$I < $splitTasks} {incr I} {
	    catch {file delete -force $remoteWorkDir/$I}
	}
    }
    localRun {
	#------ This branch is for running on local host (if you run GUI on cluster already):
	if $verbose {puts stdout "Starting RMD calculations..."}
	if [catch {Fit_CalculateResponseMatrixDerivative -varList $varList \
		       -splitTasks $splitTasks -verbose $verbose \
		       -scriptName $scriptName -scriptParameters $scriptParameters \
		       -matrixFile $matrixFile -tmpDirName $queueDirName -rootTaskName $rootTaskName \
		       -useQsub $useQsub -continuePrevious $continuePrevious -usePopupWindow 0 \
		       -waitTime $waitTime -qsubCommand $qsubCommand -qsubRespProcCommand $qsubRespProcCommand \
		       -waitInterval $waitInterval -submissionPause $submissionPause -queueSystemName $queueSystemName \
		       -abortFile [APSTmpString]} result] {
	    ReportErrorAndStop -errorFile $errorFile -stopMessage "Fit_CalculateResponseMatrixDerivative: $result"
	}
	if [catch {ChangeColumnNames -matrixFile $matrixFile \
		       -colPrefix $colPrefix -colSuffix $colSuffix} result] {
	    return -code error "ChangeColumnNames: $result"
	}

	if [catch {open $doneFile w} fid] { 
	    ReportErrorAndStop -errorFile $errorFile -stopMessage "$fid"
	} else  {
	    puts $fid "Done."
	    close $fid
	}
    }
    default {
	ReportErrorAndStop -errorFile $errorFile -stopMessage "Wrong mode: $mode"
    }
}

exit
