# ------ MAIN PROGRAM STARTS HERE -------------------------------------------------------------------------------------------
# ------ MAIN PROGRAM STARTS HERE -------------------------------------------------------------------------------------------
# ------ MAIN PROGRAM STARTS HERE -------------------------------------------------------------------------------------------

proc CalculateLOCOFit { args } {
    
    global latticeIterFile latticeRMFile lteFile optimFile
    global dispersionFile variableFile
    global SVnumber badPointsLevel badPointsIterCounter bpmTiltIterCounter
    global rmDerivFile rmDerivInverseFile rmDifferenceFile twiFile
    global hrmMeasured vrmMeasured hrmIterFile vrmIterFile hrmDifference vrmDifference 
    global elementListVar elementListRM
    global hYes vYes nuxMeasured nuyMeasured corFraction solutionParamFileList
    global usePreviousSolution previousResultsFile resultsFile elementsUpdateFile
    global averageErrorFile allElementsFile specialElementsFile specialParamFile
    global matlabScriptDir matlabPipeFile matlabLogFile matlabErrorFile matlabStarted svdProgramName
    global useQsubLong optimNonlin makeBpmNamesUnique makeSkewNamesUnique
    global workDir tmpDir definitionFile verbose useQuadConstraints quadConstraintWeight
    global outputStatusDevice iterationsLogFile convergenceTolerance resErrorMax quadConstraintFile
    global convergenceTolerance2 iterations useParallelMultiplication matrixDivisions

    set time1 [exec date +%s]
    set matlabStarted 0
    set requiredVarList [list latticeInputFile xRawResponse yRawResponse \
			     averLevel workDir beamlineName \
			     elementListVar elementListRM optionsList nuxMeasured nuyMeasured useMeasuredDisp \
			     usePreviousSolution recalculateDerivative recalculateInverse \
			     continuePrevious nonlinLevel badPointsLevel useBpmWeight recombineDerivative \
			     SVnumber corFraction iterations resultsFile coupledMatrix filterDerivative \
			     definitionFile makeBpmNamesUnique acceleratorCode calculateDerivOnly \
			     outputStatusDevice bpmTiltLevel useKickFiles abortFile verbose]

    set resErrorList ""
    foreach var $requiredVarList {set $var ""}
    set args1 $args

    APSParseArguments $requiredVarList
    set missingArgs ""
    foreach var $requiredVarList {
	if ![string length [set $var]] {lappend missingArgs $var}
    }
    if [llength $missingArgs] {
	return -code error "There are missing arguments: $missingArgs"
    }
    if {![string length $optionsList]} {
        return -code error "No fit options given. \"options\" is empty."
    } else {
        global options
        array set options $optionsList
    }
    if {[string compare $acceleratorCode elegant] == 0} {set rawLteFile $latticeInputFile; set optimFile dummy} \
	else {set optimFile $latticeInputFile}

    #------ Not required variables:
    set nonrequiredVarList [list xyRawResponse yxRawResponse dispRawFile previousResultsFile measuredFormat kickFileX kickFileY \
				rawRMDeriv secondRun latticeParamFileList prepareMatricesOnly]
    foreach var $nonrequiredVarList {set $var ""}
    set measuredFormat 1
    set secondRun 0
    set prepareMatricesOnly 0
    APSParseArguments $nonrequiredVarList
    #------ Make unique BPM names:
    if $makeBpmNamesUnique {
	if {[string first uniqueBpms $rawLteFile] != -1} {
	    puts stdout "GOTCHA -- uniqueBpms.uniqueBpms -- GOTCHA"
	    puts stdout "GOTCHA -- uniqueBpms.uniqueBpms -- GOTCHA"
	    puts stdout "GOTCHA -- uniqueBpms.uniqueBpms -- GOTCHA"
	}
	set lteFile $workDir/[file tail $rawLteFile].uniqueBpms
	if [catch {Fit_MakeBpmNamesUnique -inputLTE $rawLteFile -outputLTE $lteFile} result] {
	    return -code error "Fit_MakeBpmNamesUnique: $result"
	}
    } else {
	set lteFile $rawLteFile
    }

    #------ definitionFile defines a lot of internal variables.
    if ![string length $definitionFile] {
	set definitionFile $workDir/initialDefinitions.tcl
	uplevel \#0 "set definitionFile $workDir/initialDefinitions.tcl"
    }
    if [catch {Fit_ReadDefinitionFile -definitionFile $definitionFile -workDir $workDir} result] {
	return -code error "Fit_ReadDefinitionFile: $result"
    }
    global closedOrbit
    if {!$closedOrbit && $options(useEnergy)} {
	OutputStatusMessage "Warning: Energy fitting must be disabled when using non-closed trajectories. Disabling..."
	set options(useEnergy) 0
    }

    #------ Read number of iterations in the first run
    if {$secondRun == 2} {
	if [catch {exec sddsprocess $iterationsLogFile.sdds -pipe=out -clip=0,1,inv \
		       | sdds2stream -pipe=in -col=Iteration} result] {
	    return -code error "Error reading $iterationsLogFile.sdds: $result"
	}
	set iterations0 [format %1.0f $result]
    } else {
	catch {file delete $iterationsLogFile}
    }

    OutputStatusMessage "Calculations started..."

    if [catch {InitialPreparations -lteFile $lteFile -optimFile $optimFile \
		   -latticeParamFileList $latticeParamFileList -measuredFormat $measuredFormat \
		   -usePreviousSolution $usePreviousSolution -previousResultsFile $previousResultsFile \
		   -useMeasuredDisp $useMeasuredDisp -coupledMatrix $coupledMatrix \
		   -useBpmWeight $useBpmWeight -xRawResponse $xRawResponse -yRawResponse $yRawResponse \
		   -xyRawResponse $xyRawResponse -yxRawResponse $yxRawResponse -beamlineName $beamlineName \
		   -dispRawFile $dispRawFile -nonlinLevel $nonlinLevel -acceleratorCode $acceleratorCode \
		   -makeBpmNamesUnique $makeBpmNamesUnique -bpmTiltLevel $bpmTiltLevel \
		   -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY \
		   -specialElementsFile $specialElementsFile -specialParamFile $specialParamFile \
		   -prepareMatricesOnly $prepareMatricesOnly} result] {
	return -code error "InitialPreparations: $result"
    }
	
    if !$prepareMatricesOnly  {
	if {[string length $nuxMeasured] && [string length $nuyMeasured]} {
	    OutputStatusMessage "[format "%s %10.4f %10.4f" "Measured Tunes are   " $nuxMeasured $nuyMeasured]"
	} else {
	    OutputStatusMessage "Warining: Measured betatron tunes are missing."
	}

	set badPointsIterCounter 0
	set bpmTiltIterCounter 0
	set residualError 10.0
	if $usePreviousSolution {
	    if ![catch {exec sdds2stream $previousResultsFile -para=ResidualError} result] {
		#------ Need lindex here because the file is multipage.
		set residualError [lindex $result 0]
	    }
	    if ![catch {exec sdds2stream $previousResultsFile -para=BadPointsIterCounter} result] {
		set badPointsIterCounter [expr int([lindex $result 0])]
	    }
	}
	if [catch {CalculateRMDifference -latticeFile $latticeIterFile -acceleratorCode $acceleratorCode \
		       -xyColumnFile $rmDifferenceFile -dispersionFile $dispersionFile -allElementsFile $allElementsFile \
		       -hrmMeasured $hrmMeasured -hrmIterFile $hrmIterFile -hrmDifference $hrmDifference \
		       -vrmMeasured $vrmMeasured -vrmIterFile $vrmIterFile -vrmDifference $vrmDifference \
		       -residualError $residualError -badPointsLevel $badPointsLevel -nonlinLevel $nonlinLevel \
		       -optimNonlin $optimNonlin -bpmTiltLevel $bpmTiltLevel -currentIteration -1 \
		       -averLevel $averLevel -averageErrorFile $averageErrorFile -coupledMatrix $coupledMatrix \
		       -useQuadConstraints $useQuadConstraints -quadConstraintWeight $quadConstraintWeight \
		       -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY} rmsList] {
	    return -code error "CalculateRMDifference: $rmsList"
	}
	array set rmsArray $rmsList
	set residualError $rmsArray(rmsTotal)
	lappend resErrorList $residualError
	file copy -force $hrmIterFile $hrmIterFile.before
	file copy -force $vrmIterFile $vrmIterFile.before
	if [catch {IterationResultsOutput -rmsList $rmsList -coupledMatrix $coupledMatrix -secondRun $secondRun \
		       -fitDispersion $options(fitDispersion) -iteration 0 -sddsFile $iterationsLogFile.sdds} result] {
	    return -code error "IterationResultsOutput: $result"
	}
	
	if [file exists $abortFile] {
	    return -code error "Iterrupted by user."
	}

	#------ Build the Response Matrix Derivative
	set rmdChanged 0
	if {$recalculateDerivative && $iterations} {
	    if [catch {BuildResponseMatrixDeriv -continuePrevious $continuePrevious -optimFile $optimFile \
			   -rmDerivFile $rmDerivFile.original -useQsubLong $useQsubLong -acceleratorCode $acceleratorCode \
			   -hrmFile $hrmIterFile.before -vrmFile $vrmIterFile.before \
			   -hrmXFile $hrmIterFile.XRM -vrmXFile $vrmIterFile.XRM -coupledMatrix $coupledMatrix \
			   -lteFile $lteFile -beamlineName $beamlineName -solutionParamFileList $solutionParamFileList \
			   -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY \
			   -abortFile $abortFile} result] {
		return -code error "BuildResponseMatrixDeriv: $result"
	    }
	    file delete -force $rmDerivFile
	    exec ln -s $rmDerivFile.original $rmDerivFile
	    set rmdChanged 1
	}
	if $calculateDerivOnly {
	    return -code ok
	}
    }
    set currentRMDFile $rmDerivFile.original

    #------ Filter out the Response Matrix Derivative
    #------ Filtering is always done on .original or rawRMDeriv
    if {$filterDerivative && $iterations} {
	if {[string length $rawRMDeriv] && [file exists $rawRMDeriv]} {
	    set currentRMDFile $rawRMDeriv
	}
        if [catch {FilterResponseMatrixDeriv -inputFile $currentRMDFile -outputFile $rmDerivFile.filtered \
                       -fitDispersion $options(fitDispersion) \
                       -elementListVar $elementListVar -elementListRM $elementListRM -optionsList $optionsList \
                       -coupledMatrix $coupledMatrix -useTune $options(useTune) \
		       -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY} result] {
            return -code error "FilterResponseMatrixDeriv: $result"
        }
	file delete -force $rmDerivFile
	exec ln -s $rmDerivFile.filtered $rmDerivFile
	set currentRMDFile $rmDerivFile.filtered
	set rmdChanged 1
    }
    
    #------ Recombine the Response Matrix Derivative with quad constraints
    #------ If rmdChanged, used currentRMDFile. If not, use source file from previous calculations.
    if {$recombineDerivative && $iterations} {
	if $useQuadConstraints {
	    if !$rmdChanged {set currentRMDFile [exec sdds2stream $rmDerivFile.QC -para=SourceFile]}
	    if [catch {RecombineResponseMatrixDeriv -input $currentRMDFile -output $rmDerivFile.QC \
			   -constraintWeight $quadConstraintWeight -constraintFile $quadConstraintFile \
			   -variableFile $variableFile -coupledMatrix $coupledMatrix \
			   -useQuadConstraints $useQuadConstraints} result] {
		return -code error "RecombineResponseMatrixDeriv: $result"
	    }
	    file delete -force $rmDerivFile
	    exec ln -s $rmDerivFile.QC $rmDerivFile
	} else {
	    #------ Recombining with useQuadConstraints=0 - removing quad constraints:
	    #------ Done only when the matrix was not recalculated:
	    if !$rmdChanged {
		set currentRMDFile [exec sdds2stream $rmDerivFile.QC -para=SourceFile]
		file delete -force $rmDerivFile
		exec ln -s $currentRMDFile $rmDerivFile
	    }
	}
    }
    
    if [file exists $abortFile] {
	return -code error "Iterrupted by user."
    }

    #------ Calculate inverse Response Matrix Derivative
    if {[lsearch -exact [list matlab octave] $svdProgramName] != -1} {
	if $iterations {
	    if [catch {CalculateSVDdecomposition -matrixFile $rmDerivFile -programName $svdProgramName} result] {
		return -code error "CalculateSVDdecomposition: $result"
	    }
	}
	if {$recalculateInverse && $iterations} {
	    if [catch {CalculateRMDerivInverse -programName $svdProgramName -SVnumber $SVnumber \
			   -rmDerivFile $rmDerivFile -rmDerivInverseFile $rmDerivInverseFile \
			   -rmDifferenceFile $rmDifferenceFile} result] {
		return -code error "CalculateRMDerivInverse: $result"
	    }
	}
    }
    
    #------ Don't use else here. svdProgramName could be redefined if matlab didn't start above.
    if ![string compare $svdProgramName "sdds"] {
        if {$recalculateInverse && $iterations} {
            if [catch {CalculateRMDerivInverse -programName $svdProgramName -SVnumber $SVnumber \
                           -rmDerivFile $rmDerivFile -rmDerivInverseFile $rmDerivInverseFile \
                           -rmDifferenceFile $rmDifferenceFile} result] {
                return -code error "CalculateRMDerivInverse: $result"
            }
        }
    }
    
    #------ Load RM Inverse into matlab memory
    if {![string compare $svdProgramName "matlab"] && $iterations} {
        if [catch {LoadRMInverse -rmDerivInverseFile $rmDerivInverseFile -recalculateInverse $recalculateInverse \
                       -workDir $workDir} result] {
            return -code error "LoadRMInverse: $result"
        }
    }

    #------ Split Inverse matrix for parallel multiplication
    if $prepareMatricesOnly {
	if [catch {Fit_ReadDefinitionFile -definitionFile $definitionFile -workDir $workDir \
		       -variableList "useParallelMultiplication matrixDivisions" -abortOnError 0} result] {
	    return -code error "Fit_ReadDefinitionFile: $result"
	}
	
    }
    if [info exists useParallelMultiplication] {
	if $useParallelMultiplication {if ![info exists matrixDivisions] {set matrixDivisions 4}}
    } else {
	set useParallelMultiplication 0
    }
    if {$useParallelMultiplication && $secondRun != 2 && $iterations != 0} {
	set doSplit 0
	if $recalculateInverse {
	    set doSplit 1
	} else {
	    for {set i 1} {$i <= $matrixDivisions} {incr i} {if ![file exists ${rmDerivInverseFile}$i] {set doSplit 1}}
	}
	if $doSplit {
	    OutputStatusMessage "[exec date +%D--%T]: Splitting derivative matrix..."
	    set rmdRows [llength [join $elementListVar ]]
	    if $options(fitDispersion) {set rmdRows [expr $rmdRows + 1]}
	    set rmdRowLimit [expr round($rmdRows / $matrixDivisions) + 1]
	    exec sddsbreak $rmDerivInverseFile -pipe=out -rowlimit=$rmdRowLimit \
		| sddssplit -pipe=in -rootname=$rmDerivInverseFile -digits=1 -ext=
	}
    }
    set time2 [exec date +%s]
    OutputStatusMessage "Preparation took [expr $time2 - $time1] sec." -logFileOnly 1
    if $prepareMatricesOnly {
	OutputStatusMessage "Done: Matrices prepared for fit."
	return
    }
#
# ------- Starting iterations: ------------------------------------------------------------------------------------------
#
    set recalcIterRMD 0
    set pwd [pwd]
    file copy -force $hrmMeasured $hrmMeasured.original
    file copy -force $vrmMeasured $vrmMeasured.original
    if [catch {GetTolerance -secondRun $secondRun} convergenceTol] {
	return -code error "GetTolerance: $convergenceTol"
    }
    
    for {set I 0} {$I < $iterations} {incr I 1} {
        
	set time1 [exec date +%s]
	if {$recalcIterRMD && ($I != 0)} {
	    if [catch {CalculateAndInvertRMD -coupledMatrix $coupledMatrix -continuePrevious $continuePrevious \
			   -acceleratorCode $acceleratorCode -abortFile $abortFile} result] {
		return -code error "CalculateAndInvertRMD: $result"
	    }
	}

	if {$secondRun == 2} {
	    OutputStatusMessage "-------> Iteration number $I ($iterations0)"
	} else {
	    OutputStatusMessage "-------> Iteration number $I "
	}
        
        #------ Matrix multiplication ---
	if ![string compare $svdProgramName "matlab"] {
            exec sddsconvert $rmDifferenceFile $workDir/matlab.xycolumn -delete=col,Rootname -delete=par,*
            exec echo "matlabMultiply;" > $matlabPipeFile
            while {[file exists $matlabPipeFile] == 1} { 
                after 500
                update
                if [file exists $matlabErrorFile] {
                    return -code error "Matlab returned error: check $matlabErrorFile file"
                }
            }
            if {![file exists $workDir/matlab.xycolumn.tmp]} {
                return -code error "Problem running matlab. Check file $matlabErrorFile"
            }
            file copy -force $workDir/matlab.xycolumn.tmp $workDir/vector_out
        } else {
	    set time1 [exec date +%s]
	    if $useParallelMultiplication {
		if [catch {ParallelMultiplication -matrix1 $rmDerivInverseFile -matrix2 $rmDifferenceFile \
			       -matrixDivisions $matrixDivisions -output $workDir/vector_out} result] {
		    return -code error "ParallelMultiplication: $result"
		}
	    } else {
		exec sddsmatrixmult $rmDerivInverseFile $rmDifferenceFile $workDir/vector_out
	    }
	    set time2 [exec date +%s]
	    OutputStatusMessage "Main Loop:: sddsmatrixmult took [expr $time2 - $time1] sec." -logFileOnly 1
        }
        set vectorStdDev [exec sddsprocess $workDir/vector_out -pipe=out -process=MatrixDifference,standardDeviation,Std \
			      | sdds2stream -pipe=in -para=Std]
        
        #------ Updating the model after the iteration ---
        if [catch {UpdateModel -elementsUpdateFile $elementsUpdateFile -vectorOutFile $workDir/vector_out \
                       -corFraction $corFraction -variableFile $variableFile -allElementsFile $allElementsFile} result] {
            return -code error "UpdateModel: $result"
        }
        
        #------ Calculate response matrix difference ---
	set time3 [exec date +%s]
        if [catch {CalculateRMDifference -acceleratorCode $acceleratorCode \
                       -dispersionFile $dispersionFile -averageErrorFile $averageErrorFile \
                       -xyColumnFile $rmDifferenceFile -allElementsFile $allElementsFile \
                       -hrmMeasured $hrmMeasured -vrmMeasured $vrmMeasured \
                       -hrmDifference $hrmDifference -vrmDifference $vrmDifference \
                       -hrmIterFile $hrmIterFile -vrmIterFile $vrmIterFile \
                       -latticeFile $latticeIterFile -coupledMatrix $coupledMatrix \
                       -residualError $residualError -bpmTiltLevel $bpmTiltLevel \
                       -badPointsLevel $badPointsLevel -nonlinLevel $nonlinLevel -optimNonlin $optimNonlin \
                       -averLevel $averLevel -currentIteration $I \
		       -useQuadConstraints $useQuadConstraints -quadConstraintWeight $quadConstraintWeight \
		       -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY} rmsList] {
	    OutputStatusMessage "Last variable vector std: $vectorStdDev."	    
	    OutputStatusMessage "CalculateRMDifference: $rmsList"
 	    if [catch {StopExecution -resultsFile $resultsFile -allElementsFile $allElementsFile \
                           -hrmIterFile $hrmIterFile -vrmIterFile $vrmIterFile \
			   -rmsList $rmsList -badPointsIterCounter $badPointsIterCounter} result] {
                return -code error "StopExecution(1): $result"
            }
	    OutputStatusMessage "Last variable vector std: $vectorStdDev."	    
            if [catch {StopExecution -resultsFile $resultsFile -allElementsFile $allElementsFile \
                           -hrmIterFile $hrmIterFile -vrmIterFile $vrmIterFile \
			   -rmsList $rmsList -badPointsIterCounter $badPointsIterCounter} result] {
                return -code error "StopExecution(2): $result"
            }
            return -code error "CalculateRMDifference: $rmsList"
        }
	set time4 [exec date +%s]
	OutputStatusMessage "Main Loop:: Matrix difference took [expr $time4 - $time3] sec." -logFileOnly 1
	array set rmsArray $rmsList
	set residualError $rmsArray(rmsTotal)
	lappend resErrorList $residualError
        
        if [catch {IterationResultsOutput -rmsList $rmsList -vectorStdDev $vectorStdDev -iteration [expr $I + 1] \
                       -fitDispersion $options(fitDispersion) -coupledMatrix $coupledMatrix \
		       -sddsFile $iterationsLogFile.sdds -secondRun $secondRun} result] {
            return -code error "IterationResultsOutput: $result"
        }
        
        if [file exists $abortFile] {
            if [catch {StopExecution -resultsFile $resultsFile -allElementsFile $allElementsFile \
                           -hrmIterFile $hrmIterFile -vrmIterFile $vrmIterFile \
			   -rmsList $rmsList -badPointsIterCounter $badPointsIterCounter} result] {
                return -code error "StopExecution(2): $result"
            }
            return -code error "Interrupted by user."
        }
	set time2 [exec date +%s]
	OutputStatusMessage "Main Loop:: Iteration $I took [expr $time2 - $time1] sec." -logFileOnly 1

	#------ Checking for convergence and for maximum error
	set convergence [expr abs([lindex $resErrorList [expr [llength $resErrorList] - 1]] - \
				      [lindex $resErrorList [expr [llength $resErrorList] - 2]])]
	if {$convergence < $convergenceTol} {
	    OutputStatusMessage "Iterations converged."
	    break
	}
	if [info exists resErrorMax] {
	    if {[lindex $resErrorList [expr [llength $resErrorList] - 1]] > $resErrorMax} {
		OutputStatusMessage "Iterations are diverging - iterations are interrupted."
		break
	    }
	}
    }
    
    if [catch {StopExecution -resultsFile $resultsFile -allElementsFile $allElementsFile \
                   -hrmIterFile $hrmIterFile -vrmIterFile $vrmIterFile \
		   -rmsList $rmsList -badPointsIterCounter $badPointsIterCounter} result] {
        return -code error "StopExecution(3): $result"
    }
    OutputStatusMessage "Iterations are done."
    return $resErrorList
}

#------------------------------------------------------------------------------------------------------------------------
proc ParallelMultiplication {args} {
    global tmpDir qsubCommandShort qsubRespProcCommandShort queueSystemNameShort submissionPause
    global qsubCommand qsubRespProcCommand queueSystemName workDir
    set qsubCommand $qsubCommandShort
    set qsubRespProcCommand $qsubRespProcCommandShort
    set queueSystemName $queueSystemNameShort
    set submissionPause 0
    APSParseArguments {matrix1 matrix2 matrixDivisions output}
    set tmpRoot $workDir/[APSTmpString]-matrMult
    set deleteFiles ""
    set commandList ""
    set jobList ""
    set workDir [file dirname $output]
    for {set i 1} {$i <= $matrixDivisions} {incr i} {
	set jobName matrixMult$i
	set doneFile $workDir/matrixMult$i.done
	catch {file delete $doneFile}
	set command "mkdir -p $tmpDir; sddsmatrixmult $matrix1$i $matrix2 $tmpRoot.$i; touch $doneFile"
	lappend commandList $command
	lappend jobNameList $jobName
	lappend doneFileList $doneFile
	lappend outputFiles $tmpRoot.$i
	lappend deleteFiles $doneFile $tmpRoot.$i
    }
    if [catch {Fit_SubmitJobs -commandList $commandList -jobNameList $jobNameList \
		   -doneFileList $doneFileList -verbose 0} returnList] {
	return -code error "Fit_SubmitJobs: $returnList"
    }
    set pidList [lindex $returnList 0]
    set logFileList [lindex $returnList 1]
    set commandFileList [lindex $returnList 2]
    set dotOFileList [lindex $returnList 3]
    set waitTime 25
    set waitInterval 1.0
    if [catch {Fit_WaitForTasks_New -pidList $pidList -jobNameList $jobNameList -commandList $commandList \
		   -logFileList $logFileList -waitTime $waitTime \
		   -doneFileList $doneFileList -waitInterval $waitInterval -usePopupWindow 0 -verbose 0 \
		   -useQsub 1 -abortFile dummyMult} result] {
	return -code error "Fit_WaitForTasks_New: $result"
    }
    #------ Sometimes files don't get written fast enough, need to check the size:
    set keepWaiting 1
    while {$keepWaiting == 1} {
	set keepWaiting 0
	foreach filename $outputFiles {if {[file size $filename] == 0} {set keepWaiting 1}}
	if $keepWaiting {after 1000; puts stdout "----------->>> Extra waiting for matrix multiplication files to appear..."}
    }
    if [catch {eval exec sddscombine $outputFiles $output -merge -overWrite} result] {
	return -code error "Error combining matrix parts ($outputFiles): $result"
    }
    eval file delete [concat $deleteFiles $commandFileList $dotOFileList]
}
#------------------------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------------------------
#------ Procedures related to initial preparation of the problem (mainly creation of files) --------------------------------
#------------------------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------------------------

proc CalculateAndInvertRMD {args} {
    global rmDerivFile useQsubLong latticeIterFile hrmIterFile vrmIterFile
    global svdProgramName workDir SVnumber rmDerivInverseFile rmDifferenceFile

    APSParseArguments {coupledMatrix continuePrevious acceleratorCode abortFile}

    #------ Build the Response Matrix Derivative 
    if [catch {BuildResponseMatrixDeriv -continuePrevious $continuePrevious \
		   -rmDerivFile $rmDerivFile -useQsubLong $useQsubLong -acceleratorCode $acceleratorCode \
		   -latticeFile $latticeIterFile -hrmFile $hrmIterFile.before -vrmFile $vrmIterFile.before \
		   -hrmXFile $hrmIterFile.XRM -vrmXFile $vrmIterFile.XRM \
		   -coupledMatrix $coupledMatrix -abortFile $abortFile} result] {
	return -code error "BuildResponseMatrixDeriv: $result"
    }

    #------ Calculate inverse Response Matrix Derivative
    if ![string compare $svdProgramName "sdds"] {
	if [catch {CalculateRMDerivInverse -programName $svdProgramName -workDir $workDir -SVnumber $SVnumber \
		       -rmDerivFile $rmDerivFile -rmDerivInverseFile $rmDerivInverseFile \
		       -rmDifferenceFile $rmDifferenceFile} result] {
	    return -code error "CalculateRMDerivInverse: $result"
	}
    }
}

#------------------------------------------------------------------------------------------------------------------------

proc InitialPreparations { args } {

    APSParseArguments {lteFile optimFile previousResultsFile usePreviousSolution coupledMatrix \
				 useBpmWeight xRawResponse yRawResponse xyRawResponse yxRawResponse useMeasuredDisp \
				 dispRawFile nonlinLevel latticeParamFileList beamlineName acceleratorCode \
				 makeBpmNamesUnique measuredFormat useKickFiles kickFileX kickFileY bpmTiltLevel \
				 specialElementsFile specialParamFile prepareMatricesOnly}

    global tmpDir useSigmaWeightFile
    global twiFile dispersionFile elementsUpdateFile variableFile elementOrderFile allElementsFile
    global hrmMeasured vrmMeasured workDir weightFile manualWeightFile weightCorrectorsFile averageErrorFile 
    global latticeIterFile latticeRMFile dispColumn dispMeasuredFile nonlinFile sigmaWeightFile
    global hYes vYes elementListRM options tevQuadsIterOptimFile indexList variableTypeList 
    global zeroBpmTol svdProgramName locoSearchString solutionParamFileList bRho

    set hYes [llength [lindex $elementListRM 0]]
    set vYes [llength [lindex $elementListRM 1]]

    if {[lsearch -exact [list matlab octave sdds] $svdProgramName] == -1} {
	return -code error "Error: wrong svdProgramName variable. Valid values are matlab, octave, sdds."
    }

    global hrmIterFile vrmIterFile
    catch {eval file delete $hrmMeasured $vrmMeasured [glob -nocomplain -- ${hrmIterFile}*] [glob -nocomplain -- ${vrmIterFile}*]}

    if [catch {CreateElementOrderFile -elementOrderFile $elementOrderFile -acceleratorCode $acceleratorCode \
                   -twiFile $twiFile -lteFile $lteFile -optimFile $optimFile -latticeParamFileList $latticeParamFileList \
                   -beamlineName $beamlineName -dispColumn $dispColumn -useKickFiles $useKickFiles \
		   -kickFileX $kickFileX -kickFileY $kickFileY} result] {
        return -code error "CreateElementOrderFile: $result"
    }
    
    #------ Create parameter files: 
    if [catch {CreateParameterFiles -twiFile $twiFile -dispersionFile $dispersionFile \
                   -elementsUpdateFile $elementsUpdateFile -allElementsFile $allElementsFile \
                   -usePreviousSolution $usePreviousSolution -previousResultsFile $previousResultsFile \
                   -useMeasuredDisp $useMeasuredDisp -variableFile $variableFile \
                   -averageErrorFile $averageErrorFile -elementOrderFile $elementOrderFile \
                   -lteFile $lteFile -optimFile $optimFile -latticeParamFileList $latticeParamFileList \
		   -beamlineName $beamlineName -coupledMatrix $coupledMatrix -dispColumn $dispColumn \
                   -nonlinFile $nonlinFile -nonlinLevel $nonlinLevel -bpmTiltLevel $bpmTiltLevel \
		   -acceleratorCode $acceleratorCode \
		   -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY \
		   -specialElementsFile $specialElementsFile -specialParamFile $specialParamFile} result] {
        return -code error "CreateParameterFiles: $result"
    }
    if $prepareMatricesOnly {return}

    #------ Transform measured "Raw" files into "elegant-like" format and sorts them according to twiFile order.
    if [catch {TransformMeasuredFiles \
                   -xRawResponse $xRawResponse -xyRawResponse $xyRawResponse -hrmMeasured $hrmMeasured \
                   -yRawResponse $yRawResponse -yxRawResponse $yxRawResponse -vrmMeasured $vrmMeasured \
                   -coupledMatrix $coupledMatrix -zeroBpmTol $zeroBpmTol \
                   -dispRawFile $dispRawFile -dispMeasuredFile $dispMeasuredFile \
                   -elementOrderFile $elementOrderFile -acceleratorCode $acceleratorCode \
		   -measuredFormat $measuredFormat -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY} result] {
        return -code error "TransformMeasuredFiles: $result"
    }
    
    switch -regexp -- $acceleratorCode {
        elegant {
            set latticeIterFile $workDir/iterations.ele
            #------ Iteration ele file --- Include all files in case the variables exist in the prevSolFile
	    set solutionParamFileList $latticeParamFileList
	    lappend solutionParamFileList $allElementsFile.Quad
	    lappend solutionParamFileList $allElementsFile.Skew
	    lappend solutionParamFileList $allElementsFile.hCorr $allElementsFile.vCorr
	    lappend solutionParamFileList $allElementsFile.hCorrTilt $allElementsFile.vCorrTilt
	    if {[exec sdds2stream $allElementsFile.special -rows=bare] != 0} {
		lappend solutionParamFileList $specialParamFile
	    }
	    #------ Making iterations file...
	    set energy [format %10.4e [expr $bRho * 299792458e-6]]
	    if [catch {Fit_GenerateElegantFileFromLTE1 -eleOutputFile $latticeIterFile \
			   -run_setup [list "lattice $lteFile use_beamline $beamlineName default_order 2 p_central_mev $energy"] \
			   -load_parameters [list "filename_list \"$solutionParamFileList\""] \
			   -twiss_output [list "filename $twiFile"] \
			   -run_control [list "n_steps 1"] \
			   -bunched_beam [list "n_particles_per_bunch 1"] \
		       } result] {
                return -code error "Fit_GenerateElegantFileFromLTE1: $result"
	    }
        }
        optim {
            set tevQuadsIterOptimFile $workDir/Qcor_par.opi.iter
            set tevCorrsIterOptimFile $workDir/Ccor_par.opi.iter
            set latticeIterFile $workDir/iterations.opt
            set latticeRMFile   $workDir/RMDeriv.opt
            #------ Create "optim-specific" files
            #------ Creating file for quadrupoles updates:
            set tmpRoot $tmpDir/[APSTmpString].updateModel
            exec sddscombine $allElementsFile.Quad $allElementsFile.Skew $tmpRoot.quads -merge -overWrite
            if [catch {TransformSddsElements2Optim -sddsFile $tmpRoot.quads -optimFile $tevQuadsIterOptimFile -mode quads} result] {
                return -code error "TransformSddsQuads2Optim: $result"
            }
            exec sddscombine $allElementsFile.hCorrTilt $allElementsFile.vCorrTilt $tmpRoot.corrs -merge -overWrite
            if [catch {TransformSddsElements2Optim -sddsFile $tmpRoot.corrs -optimFile $tevQuadsIterOptimFile -mode corrs} result] {
                return -code error "TransformSddsQuads2Optim: $result"
            }
            #------ Creating latticeIterFile for iterations:
            if [catch {Fit_AddLine -inputFile $optimFile -outputFile $tmpRoot.latt \
                           -searchString $locoSearchString \
                           -includeString "\#include $tevCorrsIterOptimFile" -substitute 0} result] {
                return -code error "Lattice file $optimFile must have \"$locoSearchString\" indicator: Fit_AddLine: $result"
            }
            if [catch {Fit_AddLine -inputFile $tmpRoot.latt -outputFile $latticeIterFile \
                           -searchString $locoSearchString \
                           -includeString "\#include $tevQuadsIterOptimFile" -substitute 0} result] {
                return -code error "Lattice file $optimFile must have \"$locoSearchString\" indicator: Fit_AddLine: $result"
            }
            file copy -force $optimFile $latticeRMFile
            catch {file delete $tmpRoot.quads $tmpRoot.corrs $tmpRoot.latt}
        }
        default {
            return -code error "Accelerator code $acceleratorCode is not known."
        }
    }
    
    #------ Takes manually created file with user-entered bpm wights and creates a file for the program's use.
    if [catch {MakeWeightFile -useBpmWeight $useBpmWeight -manualWeightFile $manualWeightFile -weightFile $weightFile \
                   -hrmFile $hrmMeasured -vrmFile $vrmMeasured -weightCorrectorsFile $weightCorrectorsFile \
                   -coupledMatrix $coupledMatrix} result] {
        return -code error "makeWeightFiles: $result"
    }
    #------ Create dispersion sigma file:
    if !$measuredFormat {set useSigmaWeightFile 0}
    if {$options(fitDispersion) && $useSigmaWeightFile} {
	if {[lsearch -exact [join [exec sddsquery -col $dispMeasuredFile] ] DispersionSigma] == -1} {
	    set useSigmaWeightFile 0
	} else {
	    if [catch {MakeSigmaWeightFiles -dispMeasuredFile $dispMeasuredFile -hrmMeasured $hrmMeasured \
			   -vrmMeasured $vrmMeasured} result] {
		return -code error "makeSigmaWeightFiles: $result"
	    }
	}
    }
}


#-------------------------------------------------------------------------------------------------------------------------
#------ This procedure reads variables from definitionFile and makes them global

proc ReadDefinitionFile { args } {
    set variableList ""
    set warnings 1
    set abortOnError 1
    APSParseArguments {definitionFile workDir variableList warnings abortOnError}

    #------ List of required variables in the definition file:
    #------ List1 is definition of internally used files:
    set requiredList1 [list dispColumn rmDerivFile rmDerivInverseFile rmDerivFileS rmDerivFileV rmDerivFileSV \
			   rmDerivFileU rmDifferenceFile dispersionFile hrmMeasured vrmMeasured hrmDifference \
			   vrmDifference hrmIterFile vrmIterFile elementsUpdateFile noCorrectorsFile weightFile \
			   manualWeightFile iterationsLogFile weightCorrectorsFile variableFile allElementsFile \
			   averageErrorFile nonlinFile elementOrderFile dispMeasuredFile rmDerivElementFile \
			   sigmaWeightFileX sigmaWeightFileY specialElementsFile specialParamFile]
    #------ List2 is various variables defined by user that are too many to input from the GUI:
    set requiredList2 [list hCalibFactor vCalibFactor dispCalibFactor dispFitWeight thetaKickX thetaKickY thetaKickD \
			   bRho zeroBpmTol kickRMD matlabScriptDir directRMCalc fixedOrbitLength \
			   offDiagWeight adjustAverageGains optimNonlin locoSearchString \
			   showMatchingMachineConfigOnly twiFile svdProgramName closedOrbit useDoubleOrbits \
			   useQsubLong useQsubShort numberOfQsubTasksLong numberOfQsubTasksShort waitTimeLong \
			   waitTimeShort waitIntervalLong waitIntervalShort qsubCommandLong qsubCommandShort \
			   queueSystemNameShort queueSystemNameLong qsubRespProcCommandShort qsubRespProcCommandLong \
			   submissionPauseLong submissionPauseShort useQuadConstraints \
			   rmdQuadDelta rmdTiltDelta tiltCorFraction remotePath verbose useSigmaWeightFile]
    set requiredList [concat $requiredList1 $requiredList2]
    #------ This list is for variables that are not required but possible:
    set possibleList [list twissRefFile twissRefElement malignElement useIntercept useSlope useCubic dontUsePrevQuadConstr \
			  convergenceTolerance convergenceTolerance0 convergenceTolerance1 convergenceTolerance2 resErrorMax \
			  useParallelMultiplication matrixDivisions]

    #------ Reading only requested variables:
    if [string length $variableList] {
	if [catch {open $definitionFile r} fid] {
	    return -code error $fid
	}
	while {([gets $fid line] >= 0) && ([llength $variableList]> 0)} {
	    if {[string compare "\#" [string index $line 0]] != 0} {
		if {[string compare [lindex $line 0] "set"] == 0} {
		    set varName [lindex $line 1]
		    set index [lsearch -exact $variableList $varName]
		    if {$index != -1} {
			set varValue [lindex $line 2]
			uplevel "set $varName $varValue"
		    }
		    set variableList [lreplace $variableList $index $index]
		}
	    }
	}
	close $fid
	if [llength $variableList] {
	    if $abortOnError {
		return -code error "Error: the following variables were not defined in $definitionFile: $variableList"
	    } else {
		OutputStatusMessage "Warning: the following variables were not defined in $definitionFile: $variableList"
	    }
	}
	return
    }

    #------ Reading the entire definition file...
    if [catch {open $definitionFile r} fid] {
	return -code error $fid
    }
    set definedList ""
    while {[gets $fid line] >= 0} {
	if {[string compare "\#" [string index $line 0]] != 0} {
	    if {[string compare [lindex $line 0] "set"] == 0} {
		global [lindex $line 1]
		lappend definedList [lindex $line 1]
	    }
	}
    }
    close $fid
    source $definitionFile
    if $useQuadConstraints {lappend requiredList quadConstraintWeight quadConstraintFile}
    if !$closedOrbit {lappend requiredList twissRefFile twissRefElement malignElement}

    #------ Looking for missing and unnecessary variables...
    set unnecList ""
    foreach var $definedList {
	set ix1 [lsearch -exact $requiredList $var]
	set ix2 [lsearch -exact $possibleList $var]
	if {$ix1 != -1} {
	    set requiredList [lreplace $requiredList $ix1 $ix1]
	}
	if {$ix1 == -1 && $ix2 == -1} {
	    lappend unnecList $var
	}
    }
    if {[llength $unnecList] && $warnings} {
	OutputStatusMessage "Warning: following variables from $definitionFile are not required: $unnecList"
    }
    if [llength $requiredList] {
	return -code error "Error: following variables are required but are not in $definitionFile: $requiredList"
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc CreateElementOrderFile {args} {

    global tmpDir options elementListRM
    APSParseArguments {elementOrderFile twiFile lteFile optimFile latticeParamFileList beamlineName dispColumn \
			    acceleratorCode useKickFiles kickFileX kickFileY}
    
    set tmpRoot $tmpDir/[APSTmpString]-elementOrder
    set sddsLatticeFile $tmpRoot.lattice.sdds
    set deleteFiles ""
    switch -regexp -- $acceleratorCode {
        elegant { 
	    set latticeOptions "-lteFile $lteFile -beamlineName $beamlineName -latticeParamFileList \"$latticeParamFileList\""
	}
        optim { 
	    set latticeOptions "-optimFile $optimFile"
	}
        default { return -code error "Accelerator code $acceleratorCode is not known." }
    }
    if [catch {eval CalculateResponseMatrix -getSDDSLattice 1 -workDir $tmpDir \
                   -sddsLatticeFile $sddsLatticeFile -acceleratorCode $acceleratorCode \
		   $latticeOptions} result] {
        return -code error "CalculateResponseMatrix: $result"
    }
    exec sddsconvert $sddsLatticeFile -pipe=out -retain=col,ElementName,ElementType \
	| sddsprocess -pipe=in $elementOrderFile "-match=col,ElementType=*KICK*,ElementType=*MON*,|"
    lappend deleteFiles $sddsLatticeFile
    set combineFiles ""
    if $options(fitDispersion) {
        exec sddsmakedataset $tmpRoot.D -col=ElementName,type=string -data=$dispColumn
	lappend combineFiles $tmpRoot.D
	lappend deleteFiles $tmpRoot.D
    }
    if $useKickFiles {
	exec sddscollapse $kickFileX -pipe=out \
	    | sddsconvert -pipe -retain=col,ColumnName -rename=col,ColumnName=ElementName \
	    | sddsprocess -pipe=in $tmpRoot.x -print=col,ElementType,MEASUREMENTX
	exec sddscollapse $kickFileY -pipe=out \
	    | sddsconvert -pipe -retain=col,ColumnName -rename=col,ColumnName=ElementName \
	    | sddsprocess -pipe=in $tmpRoot.y -print=col,ElementType,MEASUREMENTY
	lappend combineFiles $tmpRoot.x $tmpRoot.y
	lappend deleteFiles $tmpRoot.x $tmpRoot.y
    }
    if [llength $combineFiles] {
	eval exec sddscombine $elementOrderFile $combineFiles $tmpRoot -merge
	file copy -force $tmpRoot $elementOrderFile
	lappend deleteFiles $tmpRoot
    }
    eval file delete $deleteFiles
    return
    
    #------ Rebuild the file in the following order: hcorr, vcorr, hbpm, vbpm.
    set fileList ""
    set fileCounter 0
    foreach elementList $elementListRM {
        if [llength $elementList] {
            exec sddsmakedataset $tmpRoot.elements -col=ElementName,type=string -data=[join $elementList ,]
            set missingList [exec sddsselect $tmpRoot.elements $tmpRoot.twi -pipe=out -match=ElementName -invert \
                                 | sdds2stream -pipe=in -col=ElementName]
            if [llength $missingList] {
                return -code error "Error: the following elements are missing from twiss file $tmpRoot.twi: $missingList"
            }   
        } else {
            exec sddsmakedataset $tmpRoot.elements -col=ElementName,type=string -data
        }
        exec sddsxref $tmpRoot.elements $tmpRoot.twi -pipe=out -take=s -match=ElementName \
            | sddssort -pipe=in $tmpRoot.$fileCounter -col=s
        lappend fileList $tmpRoot.$fileCounter
        incr fileCounter
    }
    if $options(fitDispersion) {
        exec sddsmakedataset $tmpRoot.D -col=ElementName,type=string -data=$dispColumn
        lappend fileList $tmpRoot.D
    }
    eval exec sddscombine $fileList $elementOrderFile -merge -overWrite
    eval file delete $fileList $tmpRoot.twi $tmpRoot.elements $tmpRoot.param \
	$tmpRoot.0 $tmpRoot.1 $tmpRoot.2 $tmpRoot.3 $tmpRoot.D
}

#-------------------------------------------------------------------------------------------------------------------------
proc CompareLists {args} {
    APSParseArguments {list0 subList}
    set missingElements ""
    foreach element $subList {if {[lsearch -exact $list0 $element] == -1} {lappend missingElements $element}}
    return $missingElements
}
proc CheckElementListsSanity {args} {
    global options
    APSParseArguments {listRM listVar}
    #------ Make sure there are no correctors or bpms that are in variables list but not in RM list:
    set hcorrRMIndex 0
    set vcorrRMIndex 1
    set hcorrVarIndex 1
    set vcorrVarIndex 2
    set hcorrTiltRMIndex 0
    set vcorrTiltRMIndex 1
    set hcorrTiltVarIndex 7
    set vcorrTiltVarIndex 8
    set xBpmRMIndex 2
    set yBpmRMIndex 3
    set xBpmVarIndex 3
    set yBpmVarIndex 4
    set xBpmTiltRMIndex 2
    set yBpmTiltRMIndex 3
    set xBpmTiltVarIndex 9
    set yBpmTiltVarIndex 10
    set energyVarIndex 5
    set rootList ""
    set list0List ""
    set subListList ""
    if $options(useCorrs) {
	lappend rootList hCorr vCorr
	lappend list0List [lindex $listRM $hcorrRMIndex] [lindex $listRM $vcorrRMIndex]
	lappend subListList [lindex $listVar $hcorrVarIndex] [lindex $listVar $vcorrVarIndex]
    }
    if $options(useCorrsTilt) {
	lappend rootList hCorrTilt vCorrTilt
	lappend list0List [lindex $listRM $hcorrTiltRMIndex] [lindex $listRM $vcorrTiltRMIndex]
	lappend subListList [lindex $listVar $hcorrTiltVarIndex] [lindex $listVar $vcorrTiltVarIndex]
    }
    if $options(useBPMs) {
	lappend rootList xBpm yBpm
	lappend list0List [lindex $listRM $xBpmRMIndex] [lindex $listRM $yBpmRMIndex]
	lappend subListList [lindex $listVar $xBpmVarIndex] [lindex $listVar $yBpmVarIndex]
    }
    if $options(useBPMsTilt) {
	lappend rootList xBpmTilt yBpmTilt
	lappend list0List [lindex $listRM $xBpmTiltRMIndex] [lindex $listRM $yBpmTiltRMIndex]
	lappend subListList [lindex $listVar $xBpmTiltVarIndex] [lindex $listVar $yBpmTiltVarIndex]
    }
    if $options(useEnergy) {
	lappend rootList Energy
	lappend list0List [lindex $listRM $hcorrRMIndex]
	lappend subListList [lindex $listVar $energyVarIndex]
    }
    set error 0
    foreach list0 $list0List subList $subListList root $rootList {
	set missingElements [CompareLists -list0 $list0 -subList $subList]
	if [string length $missingElements] {
	    OutputStatusMessage "Error: There are elements in Variables list that are not RM list ($root): $missingElements"
	    set error 1
	}
    }
    if $error {return -code error "There are missing elements"}
}
#-------------------------------------------------------------------------------------------------------------------------
#------ Creates elements files and energy.sdds and noElements.sdds and dispersion.sdds files

proc CreateParameterFiles { args } {

    APSParseArguments { twiFile dispersionFile usePreviousSolution previousResultsFile elementsUpdateFile \
                            useMeasuredDisp variableFile averageErrorFile elementOrderFile \
                            allElementsFile coupledMatrix dispColumn nonlinLevel nonlinFile \
                            lteFile optimFile latticeParamFileList beamlineName acceleratorCode \
			    useKickFiles kickFileX kickFileY bpmTiltLevel specialElementsFile specialParamFile}
    
    global options
    global workDir tmpDir
    global elementListRM elementListVar indexList variableTypeList
    set tmpRoot $tmpDir/[APSTmpString]-createParamFiles
    set deleteFiles ""
    
    catch {eval file delete [glob -nocomplain -- $allElementsFile.*] [glob -nocomplain -- $variableFile.*]}

    if [catch {CheckElementListsSanity -listRM $elementListRM -listVar $elementListVar} result] {
	return -code error "CheckElementListsSanity: $result"
    }

    set NVars 11
    set indexList [list useQuads useCorrs useCorrs useBPMs useBPMs useEnergy useSkew useCorrsTilt useCorrsTilt \
                       useBPMsTilt useBPMsTilt]
    set variableTypeList [list Quad hCorr vCorr xBpm yBpm Energy Skew hCorrTilt vCorrTilt xBpmTilt yBpmTilt]
    set parameterList [list K1 CALIBRATION CALIBRATION XCALIBRATION YCALIBRATION ENERGY TILT TILT TILT TILT TILT]
    set parameterModeList [list differential fractional fractional fractional fractional fractional differential \
                               differential differential differential differential]
    
    if $options(fitDispersion) {
        if [llength [lindex $elementListVar 1]] {
            set elementListVar [lreplace $elementListVar 1 1 [concat [lindex $elementListVar 1] $dispColumn]]
        }
    }

    set allElementsList $elementListVar
    #------ If bpm tilt fitting is "no", [xy]BpmTilts have 0 rows but we need them if bpm titls are done without fitting:
    if {!$options(useBPMsTilt) && $bpmTiltLevel != 0} {
	set allElementsList [lreplace $allElementsList 9 10 [lindex $elementListRM 2] [lindex $elementListRM 3]]
    }
    if $usePreviousSolution {
        #------ Make allElementsList which contains all present variables plus any variables from $previousResultsFile
	#------ that are not used presently
	for {set i 0} {$i < $NVars} {incr i} {
	    set pageID [lindex $variableTypeList $i]
	    if {[exec sddsprocess $previousResultsFile -pipe=out -nowarning -match=para,PageID=$pageID \
		     | sdds2stream -pipe=in -rows=bare] != 0} {
		set tmpfile $tmpRoot.$pageID.1
		set variableList [lindex $allElementsList $i]
		exec sddsmakedataset $tmpfile -col=ElementName,type=string -data=[join $variableList ,]
		lappend deleteFiles $tmpfile
		set additionalElements [join [exec sddsprocess $previousResultsFile -pipe=out -nowarning -match=para,PageID=$pageID \
						  | sddsselect -pipe $tmpfile -match=ElementName -inv \
						  | sdds2stream -pipe=in -col=ElementName] ]
		if [llength $additionalElements] {
		    set elementList [concat $variableList $additionalElements]
		    set allElementsList [lreplace $allElementsList $i $i $elementList]
		}
	    }
	}
    }

    #------ Creating zero allElements and variables files for everything regardless of options
    foreach filename [list $variableFile $allElementsFile] bigElementList [list $elementListVar $allElementsList] \
        fileList [list varFileList elemFileList] {
            set $fileList ""
            for {set i 0} {$i < $NVars} {incr i} {
                set index         [lindex $indexList $i]
                set parameter     [lindex $parameterList $i]
                set parameterMode [lindex $parameterModeList $i]
                set pageID        [lindex $variableTypeList $i]
                set elementList   [lindex $bigElementList $i]
                exec sddsmakedataset -pipe=out -col=ElementName,type=string -data=[join $elementList ,] \
                    | sddsprocess -pipe=in $filename.$pageID -def=col,ParameterValue,0.0 -print=para,PageID,$pageID \
                    -print=col,ElementParameter,$parameter -print=col,ParameterMode,$parameterMode \
                    -print=col,VariableType,$pageID
                #------ sddsmakedataset creates file with 1 row when the list is empty, now we remove this line.
                if ![llength $elementList] {
                    exec sddsprocess $filename.$pageID -nowarning -clip=1,0
                    lappend deleteFiles $filename.${pageID}~
                }
                #------ don't sort quads and skews because the issues with the OPTIM(quad name and quad variable are different!)
                if {!(![string compare $index useQuads] || ![string compare $index useSkew])} {
                    if [exec sdds2stream $filename.$pageID -rows=bare] {
                        if [catch {ResortSingleFile -orderFile $elementOrderFile -orderColumn ElementName \
                                       -file2sort $filename.$pageID -column2sort ElementName} result] {
                            return -code error "ResortSingleFile($pageID): $result"
                        }
                    }
                }
                lappend $fileList $filename.$pageID
            }
        }

    if 0 {
    #------ Creating file for special variables (for example, average gains)...
    set elementNameColumn ""
    set parameterValueColumn ""
    set elementParameterColumn ""
    set parameterModeColumn ""
    set variableTypeColumn ""
    if $options(useAverGains) {
	lappend elementNameColumn AverGainX AverGainY
	lappend parameterValueColumn 0 0
	lappend elementParameterColumn AVERGAIN AVERGAIN
	lappend parameterModeColumn differential differential
	lappend variableTypeColumn special special
    }
    exec sddsmakedataset -pipe=out \
	-col=ElementName,type=string -data=[join $elementNameColumn ,] \
	-col=ParameterValue,type=double -data=[join $parameterValueColumn ,] \
	-col=ElementParameter,type=string -data=[join $elementParameterColumn ,] \
	-col=ParameterMode,type=string -data=[join $parameterModeColumn ,] \
	-col=VariableType,type=string -data=[join $variableTypeColumn ,] \
	| sddsprocess -pipe=in $variableFile.special -print=para,PageID,special
    if ![llength $elementNameColumn] {
	#------ sddsmakedataset makes file with one line if lists are zero length. Remove it here.
	exec sddsprocess $variableFile.special -nowarning -clip=1,0
	lappend deleteFiles $variableFile.special~
    }
    }
    if [file exists $specialElementsFile] {
	if [exec sdds2stream $specialElementsFile -rows=bare] {
	    exec sddsconvert $specialElementsFile -pipe=out -retain=col,ElementName,ElementParameter,ParameterMode \
		| sddsprocess -pipe=in $variableFile.special -redef=col,ParameterValue,0 -reprint=col,VariableType,special
	} else {
	    exec sddsmakedataset $variableFile.special \
		-col=ElementName,type=string -data \
		-col=ElementParameter,type=string -data \
		-col=ParameterMode,type=string -data \
		-col=ParameterValue,type=double -data \
		-col=VariableType,type=string -data
	}
	exec sddsmakedataset $specialParamFile \
	    -col=ElementName,type=string -data \
	    -col=ElementParameter,type=string -data \
	    -col=ParameterMode,type=string -data \
	    -col=ParameterValue,type=double -data \
	    -col=VariableType,type=string -data
    } else {
	exec sddsmakedataset $variableFile.special \
	    -col=ElementName,type=string -data \
	    -col=ElementParameter,type=string -data \
	    -col=ParameterMode,type=string -data \
	    -col=ParameterValue,type=double -data \
	    -col=VariableType,type=string -data
    }
    file copy $variableFile.special $allElementsFile.special
    lappend varFileList $variableFile.special
    lappend elemFileList $allElementsFile.special

    #------ Including previous solution values into initial files...
    if $usePreviousSolution {
        foreach pageID $variableTypeList {
            set tmpFile ${tmpRoot}-prevSol.$pageID
            exec sddsprocess $previousResultsFile $tmpFile -match=par,PageID=$pageID -nowarning 
            if [exec sdds2stream -rows=bare $tmpFile] {
                exec sddsxref $allElementsFile.$pageID $tmpFile -pipe=out -nowarning -take=ParameterValue \
                    -rename=col,ParameterValue=ParameterValueOld -match=ElementName -fillIn \
                    | sddsprocess -pipe "-redef=col,ParameterValue,ParameterValue ParameterValueOld +" -nowarning \
                    | sddsconvert -pipe=in $tmpFile.1 -del=col,ParameterValueOld
                file copy -force $tmpFile.1 $allElementsFile.$pageID
                lappend deleteFiles $tmpFile $tmpFile.1
            }
        }
    }
    #------ Combining final files...
    eval exec sddscombine $varFileList $variableFile.sdds -overWrite
    eval exec sddscombine $varFileList $elementsUpdateFile -merge -overWrite
    eval exec sddscombine $elemFileList $allElementsFile.sdds -overWrite
    
    #------ Creating nonlinFiles from allElements bpm files ----
    file delete $nonlinFile.X $nonlinFile.Y
    if {$options(useBPMs) && $nonlinLevel > 0} {
        #------ This will give error if some bpms are not in fitting but in the response matrix.
        exec sddsconvert $allElementsFile.xBpm -pipe=out -retain=col,ElementName -rename=col,ElementName=BPMName \
            | sddsprocess -pipe=in $tmpRoot.nonlinX -def=col,pfitIntercept,0 -def=col,pfitSlope,0 -def=col,pfitCubic,0 \
	    -reprint=para,PageID,xBpmNonlin
        exec sddsconvert $allElementsFile.yBpm -pipe=out -retain=col,ElementName -rename=col,ElementName=BPMName \
            | sddsprocess -pipe=in $tmpRoot.nonlinY -def=col,pfitIntercept,0 -def=col,pfitSlope,0 -def=col,pfitCubic,0 \
	    -reprint=para,PageID,yBpmNonlin
        switch -exact -- $coupledMatrix {
            0 {
                file copy -force $tmpRoot.nonlinX $nonlinFile.X
                file copy -force $tmpRoot.nonlinY $nonlinFile.Y
            }
            1 {
                return -code error "Nonlin correction for coupledMatrix=1 is not done yet."
            }
            2 {
                exec sddscombine $tmpRoot.nonlinX $tmpRoot.nonlinY $nonlinFile.X -overWrite
                file copy -force $nonlinFile.X $nonlinFile.Y
            }
        }
        lappend deleteFiles $tmpRoot.nonlinX $tmpRoot.nonlinY
    }
    
    #------ Creating averageErrorFile --------------------------
    switch -exact -- $coupledMatrix {
        0 {
            set hResponseBPMs [lindex $elementListRM 2]
            set vResponseBPMs [lindex $elementListRM 3]
        }
        1 {
            set hResponseBPMs [lindex $elementListRM 3]
            set vResponseBPMs [lindex $elementListRM 2]
        }
        2 {
            set hResponseBPMs [concat [lindex $elementListRM 2] [lindex $elementListRM 3]]
            set vResponseBPMs [concat [lindex $elementListRM 2] [lindex $elementListRM 3]]
        }
    }
    foreach plane [list x y] bpmList [list $hResponseBPMs $vResponseBPMs] {
        if [llength $bpmList] {
            exec sddsmakedataset $averageErrorFile.$plane -col=BPMName,type=string -data=[join $bpmList ,]
        } else {
            exec sddsmakedataset $averageErrorFile.$plane -col=BPMName,type=string -data
        }
        if [catch {ResortSingleFile -orderFile $elementOrderFile -orderColumn ElementName \
                       -file2sort $averageErrorFile.$plane -column2sort BPMName} result] {
            return -code error "ResortSingleFile: $result"
        }
    }
    exec sddscombine $averageErrorFile.x $averageErrorFile.y -pipe=out \
        | sddsprocess -pipe=in $averageErrorFile -def=col,AverageError,0 -nowarning
    lappend deleteFiles $averageErrorFile.x $averageErrorFile.y
    
    #------ Creating dispersionFile ----------------------------------------------------
    if $options(useEnergy) {
        switch -regexp -- $acceleratorCode {
            elegant {
		set latticeOptions \
		    "-lteFile $lteFile -beamlineName $beamlineName -latticeParamFileList \"$latticeParamFileList\""
            } 
            optim {
		set latticeOptions "-optimFile $optimFile"
            }
            default {
                return -code error "Accelerator code $acceleratorCode is not known."
            }
        }
        if [catch {eval CalculateResponseMatrix -getBetaFunctions 1 -workDir $workDir \
                       $latticeOptions -twiFile $twiFile \
		       -acceleratorCode $acceleratorCode } result] {
            return -code error "CalculateResponseMatrix: $result"
        }
        #------ Making files with bpms used in fitting
        set fileList ""
        foreach etaColumn [list etax etay] plane [list X Y] bpmFile [list $allElementsFile.xBpm $allElementsFile.yBpm] {
            exec sddsconvert $twiFile -pipe=out -retain=col,ElementName,$etaColumn -rename=col,$etaColumn=$dispColumn \
                | sddsprocess -pipe -print=para,Plane,$plane "-redef=col,$dispColumn,$dispColumn 1000 *,units=mm" \
                | sddsselect -pipe=in $bpmFile $tmpRoot-energy.$plane -match=ElementName
            lappend fileList $tmpRoot-energy.$plane
        }
        eval exec sddscombine $fileList $dispersionFile -overWrite
        set deleteFiles [concat $deleteFiles $fileList]
    }
    eval file delete $deleteFiles
}

#-------------------------------------------------------------------------------------------------------------------------
# This procedures resort variableFile according to orderFile.

proc ResortSingleFile {args} {
    global tmpDir
    APSParseArguments { orderFile orderColumn file2sort column2sort }
    set tmpRoot $tmpDir/[APSTmpString]-resort
    set rowsBefore [exec sdds2stream $file2sort -rows=bare]
    if {[lsearch -exact [exec sddsquery -col $orderFile] Index] == -1 } {
        exec sddsprocess $orderFile $tmpRoot.index -def=col,Index,i_row
    } else {
        file copy -force $orderFile $tmpRoot.index
    }
    exec sddsxref $file2sort $tmpRoot.index -pipe=out -take=Index -match=$column2sort=$orderColumn -nowarning \
        | sddssort -pipe -col=Index \
        | sddsconvert -pipe=in $tmpRoot.sorted -del=col,Index
    set rowsAfter [exec sdds2stream $tmpRoot.sorted -rows=bare]
    if {$rowsBefore != $rowsAfter} {
        file copy $file2sort $tmpRoot.beforeSorting
        OutputStatusMessage "Warning: ResortSingleFile: Number of rows if file $file2sort before the sorting \
                                       ($rowsBefore) is different from after the sorting ($rowsAfter). The original file \
                                       $file2sort was copied to $tmpRoot.beforeSorting."
    }
    file copy -force $tmpRoot.sorted $file2sort
    file delete $tmpRoot.index $tmpRoot.sorted
}

#-------------------------------------------------------------------------------------------------------------------------

proc TransformMeasuredFiles { args } {
    
    APSParseArguments { xRawResponse xyRawResponse hrmMeasured yRawResponse yxRawResponse vrmMeasured \
                                  zeroBpmTol elementOrderFile coupledMatrix dispRawFile dispMeasuredFile acceleratorCode \
				  measuredFormat useKickFiles kickFileX kickFileY}
    
    global tmpDir elementListRM 
    global hCalibFactor vCalibFactor dispCalibFactor dispFitWeight
    global options dispColumn makeBpmNamesUnique machineName
    
    set tmpRoot $tmpDir/[APSTmpString]-transfMeas
    set deleteFiles ""

    if $useKickFiles {
	set corrXList [join [exec sddscollapse $kickFileX -pipe=out | sdds2stream -pipe=in -col=ColumnName] ]
	set corrYList [join [exec sddscollapse $kickFileY -pipe=out | sdds2stream -pipe=in -col=ColumnName] ]
    } else {
	set corrXList [lindex $elementListRM 0]
	set corrYList [lindex $elementListRM 1]
    }
    set bpmXList  [lindex $elementListRM 2]
    set bpmYList  [lindex $elementListRM 3]
    
    if {[lsearch -exact [join [exec sddsquery $xRawResponse -para] ] MeasuredFormat] != -1} {
	set measuredFormat [lindex [exec sdds2stream $xRawResponse -para=MeasuredFormat] 0]
    }
    #------ If input files are already in elegant form (simulation of correction):
    if {$measuredFormat == 0} {
	#------ Filtering unused elements:
	if $options(fitDispersion) {
	    set corrXList [concat $corrXList $dispColumn]
	}
	exec sddsmakedataset $tmpRoot.bpmsx -col=BPMName,type=string -data=[join $bpmXList ,]
	exec sddsmakedataset $tmpRoot.bpmsy -col=BPMName,type=string -data=[join $bpmYList ,]
	exec sddscombine $tmpRoot.bpmsx $tmpRoot.bpmsy $tmpRoot.bpms -overWrite
	lappend deleteFiles $tmpRoot.bpmsx $tmpRoot.bpmsy $tmpRoot.bpms
	exec sddsconvert $xRawResponse -pipe=out -retain=column,BPMName,[join $corrXList ,] \
	    | sddsselect -pipe=in $tmpRoot.bpms $xRawResponse.filtered -match=BPMName
	exec sddsconvert $yRawResponse -pipe=out -retain=column,BPMName,[join $corrYList ,] \
	    | sddsselect -pipe=in $tmpRoot.bpms $yRawResponse.filtered -match=BPMName
	lappend deleteFiles $xRawResponse.filtered $yRawResponse.filtered
	#------ Keeping required pages in calculated files:
	switch -regexp -- $coupledMatrix {
	    0 {
		exec sddsprocess $xRawResponse.filtered $hrmMeasured -match=para,Plane=X
		exec sddsprocess $yRawResponse.filtered $vrmMeasured -match=para,Plane=Y
	    }
	    2 {
		file copy -force $xRawResponse.filtered $hrmMeasured
		file copy -force $yRawResponse.filtered $vrmMeasured
	    }
	}
	eval file delete $deleteFiles
	return
    }

    #------ Here the calibration factors depend only on the plane of steering magnets. If they are also dependent on the bpms,
    #------ then it must be taken into account in advance. It is important for coupled solution.
    
    if $options(fitDispersion) {
        if ![file exists $dispRawFile]  {
            return -code error "Measured dispersion file is missing: $dispRawFile ."
        }
        #------ Transform dispersion measurement into 2-page 2-column (BPMName, Dispersion) file.
        switch -regexp -- $machineName {
            APS {
                exec sddsconvert $dispRawFile -pipe=out -retain=col,Rootname,xDispersion,xDispersionSigma \
                    -rename=col,xDispersion=Dispersion -rename=col,xDispersionSigma=DispersionSigma \
                    | sddsprocess -pipe=in $tmpRoot.X -print=para,Plane,X
                exec sddsconvert $dispRawFile -pipe=out -retain=col,Rootname,yDispersion,yDispersionSigma \
                    -rename=col,yDispersion=Dispersion -rename=col,yDispersionSigma=DispersionSigma \
                    | sddsprocess -pipe=in $tmpRoot.Y -print=para,Plane,Y
                exec sddscombine $tmpRoot.X $tmpRoot.Y -pipe=out \
                    | sddsconvert -pipe=in $dispMeasuredFile -rename=col,Rootname=BPMName
		if {[lsearch -exact [join [exec sddsquery -col $dispMeasuredFile] ] DispersionSigma] != -1} {
		    exec sddsprocess $dispMeasuredFile $tmpRoot.weight "-redef=col,DispersionSigma,DispersionSigma abs"
		    file copy -force $tmpRoot.weight $dispMeasuredFile
		    lappend deleteFiles $tmpRoot.weight
		}
                lappend deleteFiles $tmpRoot.X $tmpRoot.Y
            }
            TeVatron|Debuncher|Accumulator|Recycler|TAA|generic {
                file copy -force $dispRawFile $dispMeasuredFile
            }
            default {
                return -code error "Machine name $machineName is not known."
            }	
        }
    }
    
    switch -exact -- $coupledMatrix {
        0 {
            set planeFlagList [list [llength $corrXList] [llength $corrYList]]
            set correctorList [list $corrXList $corrYList]
            set measuredFileList [list $hrmMeasured $vrmMeasured]
            set calibFactorList [list $hCalibFactor $vCalibFactor]
            set rawFileList [list $xRawResponse $yRawResponse]
            set responseBPMList [list $bpmXList $bpmYList]
            set corPlaneList [list X Y]
            set bpmPlaneList [list X Y]
        }
        1 {
            set planeFlagList [list [llength $corrXList] [llength $corrYList]]
            set correctorList [list $corrXList $corrYList]
            set measuredFileList [list $hrmMeasured $vrmMeasured]
            set calibFactorList [list $hCalibFactor $vCalibFactor]
            set rawFileList [list $xyRawResponse $yxRawResponse]
            set responseBPMList [list $bpmYList $bpmXList]
            set corPlaneList [list X Y]
            set bpmPlaneList [list Y X]
        }
        2 {
            set planeFlagList [list [llength $corrXList] [llength $corrXList] [llength $corrYList] [llength $corrYList]]
            set correctorList [list $corrXList $corrXList $corrYList $corrYList]
            set measuredFileList [list $hrmMeasured.page1 $hrmMeasured.page2 $vrmMeasured.page1 $vrmMeasured.page2]
            set calibFactorList [list $hCalibFactor $hCalibFactor $vCalibFactor $vCalibFactor]
            set rawFileList [list $xRawResponse $xyRawResponse $yxRawResponse $yRawResponse]
            set responseBPMList [list $bpmXList $bpmYList $bpmXList $bpmYList]
            set corPlaneList [list X X Y Y]
            set bpmPlaneList [list X Y X Y]
            if $options(fitDispersion) {
                if {[llength [exec sdds2stream $dispMeasuredFile -rows=bare]] != 2} {
                    return -code error "Dispersion is supposed to be 2-page file!"
                } 
            }
        }
    }

    if $makeBpmNamesUnique {
	set updatedRawFileList ""
	set fileCounter 1
	foreach rawFile $rawFileList plane $bpmPlaneList {
	    if {[lsearch -exact [exec sddsquery $rawFile -col] Corrector] != -1} {
		exec sddsconvert $rawFile $tmpRoot.tmp -retain=col,Corrector
		exec sddsconvert $rawFile -pipe=out -del=col,Corrector "-edit=col,*,e i%$plane%" \
		    | sddsxref -pipe=in $tmpRoot.tmp $tmpRoot.$fileCounter -take=Corrector
		lappend deleteFiles $tmpRoot.tmp
	    } else {
		exec sddsprocess $rawFile $tmpRoot.$fileCounter "-reedit=col,BPMName,e i%$plane%"
	    }
	    lappend updatedRawFileList $tmpRoot.$fileCounter
	    incr fileCounter
	}
	exec sddsprocess $dispMeasuredFile -nowarning "-reprint=col,BPMName,%s%s,BPMName,Plane"
	lappend deleteFiles ${dispMeasuredFile}~
	set deleteFiles [concat $deleteFiles $updatedRawFileList]
    } else {
	set updatedRawFileList $rawFileList
    }
    
    foreach planeFlag $planeFlagList rawFile $updatedRawFileList responseBPMs $responseBPMList measuredFile $measuredFileList \
        corrList $correctorList calibFactor $calibFactorList plane $corPlaneList {
            if $planeFlag {
                #------ Filtering out non-used elements and multiplying with calibration coefficients.
                switch -regexp -- $machineName {
                    TeVatron|Accumulator|Debuncher|Recycler|APSBooster|generic {
                        if [catch {exec sddsconvert $rawFile -pipe=out -retain=column,BPMName,[join $corrList ,] \
				       | sddstranspose -pipe -oldColumnNames=Corrector -newColumnName=BPMName \
				       | sddsconvert -pipe -retain=column,Corrector,[join $responseBPMs ,] \
				       | sddstranspose -pipe -oldColumnNames=BPMName -newColumnName=Corrector \
				       | sddsprocess -pipe=in $measuredFile -reprint=para,Plane,$plane \
				       "-redefine=col,%s,%s $calibFactor /,select=*,exclude=BPMName,units=m/rad" \
				       -redef=para,CalibFactor,$calibFactor} result] {
			    return -code error \
				"Error filtering elements in use (rawFile: $rawFile; responseBPMs: $responseBPMs): $result"
			}
                    }
                    APS|TAA {
			#------ Response is supplied in mm/A, therefore the calibFactor is in 1000*rad/A
			if [catch {exec sddsconvert $rawFile -pipe=out -retain=column,Corrector,[join $responseBPMs ,] \
				       | sddstranspose -pipe -oldColumnNames=BPMName -newColumnName=Corrector \
				       | sddsconvert -pipe -retain=column,BPMName,[join $corrList ,] \
				       | sddsprocess -pipe=in $measuredFile -reprint=para,Plane,$plane \
				       "-redefine=col,%s,%s $calibFactor /,select=*,exclude=BPMName,units=m/rad"} result] {
			    return -code error \
				"Error filtering elements in use (rawFile: $rawFile; responseBPMs: $responseBPMs): $result"
			}
                    }
                    default {
			return -code error "Machine name $machineName is not known."
                    }
                }
                #------ Resorting rows (bpms)
                if [catch {ResortSingleFile -orderFile $elementOrderFile -orderColumn ElementName \
                               -file2sort $measuredFile -column2sort BPMName} result] {
                    return -code error "ResortSingleFile(1): $result"
                }
                #------ Resorting columns (correctors)
                exec sddstranspose $measuredFile $measuredFile.tmp -newColumn=BPMName -oldColumn=Corrector
                if [catch {ResortSingleFile -orderFile $elementOrderFile -orderColumn ElementName \
                               -file2sort $measuredFile.tmp -column2sort Corrector} result] {
                    return -code error "ResortSingleFile(2): $result"
                }
                exec sddstranspose $measuredFile.tmp $measuredFile -newColumn=Corrector -oldColumn=BPMName
                lappend deleteFiles $measuredFile.tmp
                #------ Checking for "zero" bpms
                if [catch {Fit_GetZeroBpms -filename $measuredFile -threshold $zeroBpmTol -column BPMName} zeroBpms] {
                    return -code error "Fit_GetZeroBpms: $zeroBpms"
                }
                if [llength $zeroBpms] {
		    if {[llength $zeroBpms] < 10} {
			OutputStatusMessage "TransformMeasuredFiles: Warning: the following bpms in $plane \
                            have zero readings only: [join $zeroBpms ,]"
		    } else {
			OutputStatusMessage "TransformMeasuredFiles: Warning: the following bpms in $plane \
                            have zero readings only: [join $zeroBpms ,]" -logFileOnly 1
			OutputStatusMessage "TransformMeasuredFiles: Warning: some bpms in $plane have zero readings only"
		    }
                }
            }
        }

    if {$coupledMatrix == 2} {
        exec sddscombine $hrmMeasured.page1 $hrmMeasured.page2 $hrmMeasured -overWrite
        exec sddscombine $vrmMeasured.page1 $vrmMeasured.page2 $vrmMeasured -overWrite
        lappend deleteFiles $hrmMeasured.page1 $hrmMeasured.page2 $vrmMeasured.page1 $vrmMeasured.page2
    }
    
    #------ Add dispersion column to hrmMeasured if fitDispersion
    if $options(fitDispersion) {
        switch -exact -- $coupledMatrix {
            0 {	exec sddsprocess $dispMeasuredFile $tmpRoot.disp -match=para,Plane=X }
            1 {	exec sddsprocess $dispMeasuredFile $tmpRoot.disp -match=para,Plane=Y }
            2 { file copy -force $dispMeasuredFile $tmpRoot.disp }
        }
        set missingBpms [exec sddsselect $hrmMeasured $tmpRoot.disp -pipe=out -match=BPMName -invert \
                             | sdds2stream -pipe=in -col=BPMName]
        if [llength $missingBpms] {
            OutputStatusMessage "TransformMeasuredFiles: Warning: these BPMs are missing from \
                             dispersion file: [join $missingBpms ]"
        }
        exec sddsxref $hrmMeasured $tmpRoot.disp -pipe=out -nowarning -take=Dispersion -fillIn \
            -rename=col,Dispersion=$dispColumn -match=BPMName \
            | sddsprocess -pipe=in $hrmMeasured.tmp \
            "-redef=col,$dispColumn,$dispColumn $dispCalibFactor * $dispFitWeight *"
        file copy -force $hrmMeasured.tmp $hrmMeasured
        lappend deleteFiles $hrmMeasured.tmp $tmpRoot.disp
    }
    catch {eval file delete $deleteFiles}
}

#-------------------------------------------------------------------------------------------------------------------------

proc TransformSddsElements2Optim {args} {
    APSParseArguments { sddsFile optimFile mode }
    file delete $optimFile
    switch -exact $mode {
	quads {
	    exec sddsprocess -nowarning $sddsFile -pipe=out \
		"-reprint=col,ElementName,\$%s = \$%s + %lf;,ElementName,ElementName,ParameterValue" \
		| sdds2stream -pipe=in -noquotes -col=ElementName \
		> $optimFile
	}
	corrs {
	    exec sddsprocess -nowarning $sddsFile -pipe=out \
		"-reprint=col,ElementName,\$a%s = \$a%s + %lf;,ElementName,ElementName,ParameterValue" \
		| sdds2stream -pipe=in -noquotes -col=ElementName \
		> $optimFile
	}
	default { return -code error "Wrong mode parameter: $mode" }
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc MakeWeightFile { args } {
    global tmpDir useBpmWeight
    APSParseArguments { useBpmWeight weightFile manualWeightFile hrmFile vrmFile weightCorrectorsFile coupledMatrix }
    if $useBpmWeight {
        if ![file exists $manualWeightFile] {
            return -code error "useBpmWeight is on, but the weight file $manualWeightFile is missing."
        }
        set tmpRoot $tmpDir/[APSTmpString]-weight
        switch -exact -- $coupledMatrix {
            0 {
                set rmFileList [list $hrmFile $vrmFile]
                set planeList [list X Y]
            }
            1 {
                set rmFileList [list $vrmFile $hrmFile]
                set planeList [list Y X]
            }
            2 {
                exec sddsconvert $hrmFile $tmpRoot.page1 -keepPage=1
                exec sddsconvert $hrmFile $tmpRoot.page2 -keepPage=2
                set rmFileList [list $tmpRoot.page1 $tmpRoot.page2]
                set planeList [list X Y]
            }
        }
        foreach rmFile $rmFileList plane $planeList {
            exec sddsprocess $manualWeightFile $tmpRoot.manual -nowarning -match=col,Plane=$plane
            if [exec sdds2stream $tmpRoot.manual -rows=bare] {
                exec sddsconvert $rmFile -pipe=out -retain=col,BPMName \
                    | sddsprocess -pipe=in $tmpRoot.$plane -redef=col,Index,i_row -del=para,*
                set missingBpms ""
                if [catch {exec sddsselect $tmpRoot.manual $tmpRoot.$plane -pipe=out -match=BPMName -invert \
                               | sdds2stream -pipe=in -col=BPMName} missingBpms] {
                    return -code error "$missingBpms"
                }
                if [llength $missingBpms] {
                    OutputStatusMessage "makeWeightFiles: Warning processing $plane: The following bpms in \
                         the weight file: [join $missingBpms ] are missing from the $rmFile file."
                }
                exec sddsselect $tmpRoot.$plane $tmpRoot.manual -pipe=out -match=BPMName \
                    | sddsxref -pipe=in $tmpRoot.manual $tmpRoot.$plane.weight -take=Weight -match=BPMName
                exec sddsselect $tmpRoot.$plane $tmpRoot.manual -pipe=out -match=BPMName -invert \
                    | sddsprocess -pipe=in $tmpRoot.$plane.noweight -def=col,Weight,1.0
                exec sddscombine $tmpRoot.$plane.noweight $tmpRoot.$plane.weight -pipe=out -merge \
                    | sddssort -pipe -col=Index,increasing \
                    | sddsprocess -pipe -reprint=para,Plane,$plane \
                    | sddsconvert -pipe=in $tmpRoot.$plane -del=col,Index
                file delete $tmpRoot.$plane.weight $tmpRoot.$plane.noweight
            } else {
                exec sddsconvert $rmFile -pipe=out -retain=col,BPMName \
                    | sddsprocess -pipe=in $tmpRoot.$plane -def=col,Weight,1.0 -reprint=para,Plane,$plane
            }
            file delete $tmpRoot.manual
        }
        exec sddscombine -overWrite $tmpRoot.X $tmpRoot.Y $weightFile
        file delete $tmpRoot.X $tmpRoot.Y $tmpRoot.page1 $tmpRoot.page2
    }
    if [file exists $weightCorrectorsFile] {
        OutputStatusMessage "makeWeightFiles: Warning, corrector weight are not working now!!!"
    } 
}

#-------------------------------------------------------------------------------------------------------------------------

proc RemoveAllSpaces {line} {
    set strLength [string length $line]
    set line1 ""
    for {set I 0} {$I < $strLength} {incr I} {
	if {[string compare " " [string range $line $I $I]] != 0} {
	    append line1 [string range $line $I $I]
	}
    }
    return $line1
}

#-------------------------------------------------------------------------------------------------------------------------

proc MakeBpmNamesUnique {args} {
    APSParseArguments {inputLTE outputLTE}

    if [catch {open $inputLTE r} fileId] {
	return -code error "Cannot open the file $inputLTE for reading: $fileId"
    }
    if [catch {open $outputLTE w} fileId1] {
	return -code error "Cannot open the file $outputLTE for writing: $fileId1"
    }
    while {[gets $fileId line] >= 0} {
	if {[string first "MONI" $line] != -1} {
	    set line [RemoveAllSpaces $line]
	    if {[string first "LINE=" $line] == -1} {
		set strLength [string length $line]
		if {[string compare "\"" [string range $line 0 0]] == 0} {
		    set I1 [expr 1 + [string first "\"" [string range $line 1 $strLength]]]
		    set colIndex [expr $I1 + [string first ":" [string range $line $I1 $strLength]]]
		    set elementName [string range $line 1 [expr $I1 - 1]]
		} else {
		    set colIndex [string first ":" $line]
		    set elementName [string range $line 0 [expr $colIndex - 1]]
		}
		if {[string first "MONI" [string range $line $colIndex $strLength]] != -1} {
		    puts $fileId1 "\"${elementName}X\": HMON"
		    puts $fileId1 "\"${elementName}Y\": VMON"
		    puts $fileId1 "\"${elementName}\":  LINE=(${elementName}X,${elementName}Y)"
		}
	    } else {
		puts $fileId1 $line
	    }
	} else {
	    puts $fileId1 $line
	}
    }
    close $fileId
    close $fileId1
}

#-------------------------------------------------------------------------------------------------------------------------

proc MakeSigmaWeightFiles {args} {
    global tmpDir dispColumn options sigmaWeightFileX sigmaWeightFileY
    APSParseArguments {dispMeasuredFile hrmMeasured vrmMeasured}
    set tmpRoot $tmpDir/[APSTmpString]-sigmaWeight
    set deleteFiles ""
    #------ This part makes matrix that has 1.0 everywhere except in the dispColumn, where it has weights
    if $options(fitDispersion) {
	#------ dispMeasuredFile has DispersionSigma column.
	if [catch {exec sddsoutlier $dispMeasuredFile -pipe=out -col=DispersionSigma -stdev=3 \
		       | sddsoutlier -pipe -col=DispersionSigma -stdev=3 \
		       | sddsprocess -pipe=in $tmpRoot.1 -process=DispersionSigma,average,SigmaAver} result] {
	    return -code error "Error getting average sigma: $result"
	}
	set sigmaWeightCutoff 5
	exec sddsxref $dispMeasuredFile $tmpRoot.1 -pipe=out -leave=* -transfer=para,SigmaAver \
	    | sddsprocess -pipe=in $tmpRoot.2 "-redef=col,SigmaWeight,DispersionSigma SigmaAver /" \
	    "-redef=col,SigmaWeight,SigmaWeight $sigmaWeightCutoff < ? 1.0 : SigmaWeight \$"
	lappend deleteFiles $tmpRoot.1 $tmpRoot.2
	exec sddsconvert $hrmMeasured -pipe=out -del=col,$dispColumn \
	    | sddsprocess -pipe -redef=col,%s,1.0,select=*,exclude=BPMName \
	    | sddsxref -pipe=in $tmpRoot.2 $sigmaWeightFileX -take=SigmaWeight -match=BPMName \
	    -rename=col,SigmaWeight=$dispColumn -fillin -nowarning
	exec sddsprocess $vrmMeasured $sigmaWeightFileY -redef=col,%s,1.0,select=*,exclude=BPMName
    } else {
	exec sddsprocess $hrmMeasured $sigmaWeightFileX -redef=col,%s,1.0,select=*,exclude=BPMName 
	exec sddsprocess $vrmMeasured $sigmaWeightFileY -redef=col,%s,1.0,select=*,exclude=BPMName
    }
    #------ Here do processing of RM sigmas if they ever become available...
    catch {eval file delete $deleteFiles}
}

#-------------------------------------------------------------------------------------------------------------------------

proc GetTolerance {args} {
    APSParseArguments {secondRun}
    set varList [list convergenceTolerance convergenceTolerance0 convergenceTolerance1 convergenceTolerance2]
    set tolList ""
    foreach var $varList {
	global $var
	if [info exists $var] {lappend tolList [set $var]}
    }
    set max 0
    foreach tol $tolList {if {$tol > $max} {set max $tol}}
    set min 1e10
    foreach tol $tolList {if {$tol < $min} {set min $tol}}
    if {$secondRun == 1} {set tolerance $max} else { set tolerance $min}
    return $tolerance
}

#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
#------ Procedures related to calculation of response matrices during iterations --------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------

#-------------------------------------------------------------------------------------------------------------------------
# Corrects measured matrix by removing bad points and average non-zero error (proc CorrectMeasuredResponse),
# then calculates response matrices (proc CalculateMatrices). 
# Then calculates difference with the measured ones, then applies BPM weight correction, then makes one column.

proc CalculateRMDifference { args } {
    
    set residualError 10.0
    set badPointsLevel 0
    set nonlinLevel 0
    set averLevel 0
    set averageErrorFile ""
    set coupledMatrix 0
    APSParseArguments { xyColumnFile hrmMeasured hrmIterFile hrmDifference acceleratorCode \
                                  vrmMeasured vrmDifference vrmIterFile latticeFile bpmTiltLevel \
                                  dispersionFile allElementsFile averageErrorFile coupledMatrix \
                                  residualError badPointsLevel nonlinLevel averLevel optimNonlin currentIteration \
				  useQuadConstraints quadConstraintWeight useKickFiles kickFileX kickFileY}
    
    global elementListRM tmpDir options
    
    set hYes [llength [lindex $elementListRM 0]]
    set vYes [llength [lindex $elementListRM 1]]
    
    #------ Calculate updated response matrix and make nonlinear correction.
    set time1 [exec date +%s]
    if [catch {CalculateMatrices -residualError $residualError \
                   -hrmFile $hrmIterFile -vrmFile $vrmIterFile -nonlinLevel $nonlinLevel -optimNonlin $optimNonlin \
                   -hrmMeasured $hrmMeasured -vrmMeasured $vrmMeasured \
                   -dispersionFile $dispersionFile -allElementsFile $allElementsFile -bpmTiltLevel $bpmTiltLevel \
                   -coupledMatrix $coupledMatrix -acceleratorCode $acceleratorCode -currentIteration $currentIteration \
		   -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY} result] {
        return -code error "CalculateMatrices: $result"
    }
    set time2 [exec date +%s]
    OutputStatusMessage "CalculateMatrices took [expr $time2 - $time1] sec." -logFileOnly 1
    
    #------ Correct measured response matrix: Removing bad points and subtracting average.
    set time1 [exec date +%s]
    if [catch {CorrectMeasuredResponse -hYes $hYes -vYes $vYes -hrmIterFile $hrmIterFile -vrmIterFile $vrmIterFile \
                   -hrmMeasured $hrmMeasured -vrmMeasured $vrmMeasured -badPointsLevel $badPointsLevel \
                   -averLevel $averLevel -averageErrorFile $averageErrorFile \
                   -residualError $residualError} result] {
        return -code error "CorrectMeasuredResponse: $result"
    }
    set time2 [exec date +%s]
    OutputStatusMessage "CorrectMeasuredResponse took [expr $time2 - $time1] sec." -logFileOnly 1
    
    #------ Calculate difference between measured and calculated matrices and combine into a single column.
    set time1 [exec date +%s]
    if [catch {SubtractMatricesAndMakeOneColumn -hYes $hYes -vYes $vYes -coupledMatrix $coupledMatrix \
                   -hrmMeasured $hrmMeasured -vrmMeasured $vrmMeasured \
                   -hrmIterFile $hrmIterFile -vrmIterFile $vrmIterFile -hrmDifference $hrmDifference \
                   -vrmDifference $vrmDifference -xyColumnFile $xyColumnFile \
		   -useQuadConstraints $useQuadConstraints -quadConstraintWeight $quadConstraintWeight} rmsList] {
        return -code error "SubtractMatricesAndMakeOneColumn: $rmsList"
    }
    set time2 [exec date +%s]
    OutputStatusMessage "SubtractMatricesAndMakeOneColumn took [expr $time2 - $time1] sec." -logFileOnly 1
    
    exec sddsconvert $xyColumnFile -nowarning -retain=para,ConstraintLines
    file delete ${xyColumnFile}~
    return $rmsList
}

#-------------------------------------------------------------------------------------------------------------------------
# This procedure removes bad points and subtracts averages.

proc CorrectMeasuredResponse {args} {
    
    APSParseArguments { hYes vYes hrmIterFile vrmIterFile hrmMeasured vrmMeasured badPointsLevel \
                            averLevel averageErrorFile residualError } 
    
    global badPointsIterCounter
    if ![info exists badPointsIterCounter] {set badPointsIterCounter 0}
    
    #------ Removing bad points ------------------------------
    if {$residualError < $badPointsLevel} {
        incr badPointsIterCounter
        switch -exact -- $badPointsIterCounter {
            1       {set NSigma 5}
            2       {set NSigma 4}
            default {set NSigma 3}
        }
        
        set xPoints ""
        set yPoints ""
        if $hYes {
            if [catch {RemoveBadPoints -computedResponseFile $hrmIterFile \
                           -measuredResponseFile $hrmMeasured -NSigma $NSigma \
                           -outputFile $hrmMeasured} xPoints] {
                return -code error "RemoveBadPoints: $xPoints"
            }
        }
        if $vYes {
            if [catch {RemoveBadPoints -computedResponseFile $vrmIterFile \
                           -measuredResponseFile $vrmMeasured -NSigma $NSigma \
                           -outputFile $vrmMeasured} yPoints] {
                return -code error "RemoveBadPoints: $yPoints"
            }
        }
	set pointsRemoved [expr [lindex $xPoints 0] + [lindex $yPoints 0]]
	set pointsRemovedTotal [expr [lindex $xPoints 1] + [lindex $yPoints 1]]
        OutputStatusMessage \
	    "RemoveBadPoints ($NSigma sigmas): [format %.0f $pointsRemoved] bad points removed (total: [format %.0f $pointsRemovedTotal])."
    }
    
    #------ Subtracting average -------------------------------
    if {$residualError < $averLevel} {
        if $hYes {
            if [catch {SubtractAverageError -computedResponseFile $hrmIterFile \
                           -measuredResponseFile $hrmMeasured -averageErrorFile $averageErrorFile \
                           -plane X} result] {
                return -code error "SubtractAverageError: $result"
            }
        }
        if $vYes {
            if [catch {SubtractAverageError -computedResponseFile $vrmIterFile \
                           -measuredResponseFile $vrmMeasured -averageErrorFile $averageErrorFile \
                           -plane Y} result] {
                return -code error "SubtractAverageError: $result"
            }
        }
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc RemoveBadPoints { args } {

    global tmpDir workDir dispColumn
    
    APSParseArguments { measuredResponseFile computedResponseFile outputFile NSigma }
    
    set tmpRoot $tmpDir/[APSTmpString]-removeBadPoints
    set deleteFiles ""
    #------ $dispColumn is removed if exists and merge if two pages...
    exec sddsconvert $measuredResponseFile $tmpRoot.measRM -del=col,$dispColumn
    exec sddsconvert $computedResponseFile $tmpRoot.compRM -del=col,$dispColumn
    set twoPages 0
    if {[exec sdds2stream $computedResponseFile -npages=bare] == 2} {
        set twoPages 1
        exec sddscombine $tmpRoot.measRM $tmpRoot.measRM.tmp -merge
        exec sddscombine $tmpRoot.compRM $tmpRoot.compRM.tmp -merge
        file copy -force $tmpRoot.measRM.tmp $tmpRoot.measRM
        file copy -force $tmpRoot.compRM.tmp $tmpRoot.compRM
        file delete $tmpRoot.measRM.tmp $tmpRoot.compRM.tmp
        exec sddsprocess $computedResponseFile -pipe=out "-def=col,PageNumber,i_page" \
            | sddscombine -pipe -merge \
            | sddsconvert -pipe=in $tmpRoot.pageNumber -retain=col,PageNumber
    }
    
    #------ Create a mask file...
    set maskFile $tmpRoot.mask
    exec sddschanges $tmpRoot.measRM -pipe=out -base=$tmpRoot.compRM -copy=BPMName -change=exclude=BPMName,* \
        | sddsconvert -pipe "-edit=col,ChangeIn*,a 8d" \
        | sddstranspose -pipe -newColumn=BPMName -oldColumn=Corrector \
        | tee $maskFile.transp \
        | sddsconvert -pipe -del=col,Corrector \
        | sddsprocess -pipe "-proc=*,rms,%sRms" \
        | sddsprocess -pipe "-redef=col,%s,%s abs %sRms $NSigma * > ? 1 : 0 \$,select=*" \
        | sddsxref -pipe $maskFile.transp -take=Corrector \
        | sddstranspose -pipe=in $maskFile -newColumn=Corrector -oldColumn=BPMName
    
    #------ Get the number of bad points and the number of previously removed points:
    set nPoints [exec sddsrowstats $maskFile -pipe=out -sum=ColSum,* \
                     | sddsprocess -pipe -proc=ColSum,sum,Sum \
                     | sdds2stream -pipe=in -para=Sum] 
    if [catch {exec sddscombine $measuredResponseFile -pipe=out -merge \
		   | sdds2stream -pipe=in -para=BadPointsRemoved} nPointsTotal] {
	set nPointsTotal 0
    }
    set nPointsTotal [expr $nPointsTotal + $nPoints]

    #------ Substitute computed elements using the mask and redefining BadPointsRemoved parameter:
    file copy -force $tmpRoot.measRM $tmpRoot.measRM.copy
    eval exec sddsmatrixop $tmpRoot.measRM.copy -pipe=out -push=$tmpRoot.measRM -push=$tmpRoot.compRM \
        -subtract -push=$maskFile -multiply=hadamard -subtract -columnNames=file=$maskFile.transp,column=Corrector \
        | sddsxref -pipe $tmpRoot.measRM.copy -take=BPMName \
	| sddsprocess -pipe=in $tmpRoot.measRM -redef=para,BadPointsRemoved,$nPointsTotal
    
    #------ Split back in 2 pages...
    if $twoPages {
        exec sddsxref $tmpRoot.measRM $tmpRoot.pageNumber -pipe=out -take=PageNumber \
            | sddsbreak -pipe -change=PageNumber \
            | sddsconvert -pipe=in $tmpRoot.measRM.tmp2 -del=col,PageNumber
        file copy -force $tmpRoot.measRM.tmp2 $tmpRoot.measRM
        lappend deleteFiles $tmpRoot.measRM.tmp2 $tmpRoot.pageNumber
    }
    
    #------ Return dispColumn back:
    if {[lsearch -exact [exec sddsquery -col $measuredResponseFile] $dispColumn] != -1 } {
        exec sddsxref $tmpRoot.measRM $measuredResponseFile $tmpRoot.measRM.1 -take=$dispColumn
        file copy -force $tmpRoot.measRM.1 $outputFile
        lappend deleteFiles $tmpRoot.measRM $tmpRoot.measRM.1 $tmpRoot.compRM
    } else {
        file copy -force $tmpRoot.measRM $outputFile
        lappend deleteFiles $tmpRoot.measRM $tmpRoot.compRM
    }
    eval file delete $deleteFiles $maskFile $tmpRoot.measRM.copy $maskFile.transp

    return [list $nPoints $nPointsTotal]
}

#-------------------------------------------------------------------------------------------------------------------------
# Subtracts nonzero average from each bpm. This is supposed to take care of dispersion.

proc SubtractAverageError {args} {
    
    global tmpDir dispColumn
    
    set tmpFile $tmpDir/[APSTmpString].avError
    APSParseArguments {computedResponseFile measuredResponseFile averageErrorFile plane}
    if [catch {exec sddschanges $computedResponseFile -pipe=out -base=$measuredResponseFile \
                   -change=exclude=BPMName,* -copy=BPMName -parallelPages \
                   | sddsconvert -pipe -del=col,ChangeIn$dispColumn \
                   | sddsrowstats -pipe -mean=AverageError,ChangeIn* \
                   | sddsconvert -pipe -retain=col,BPMName -retain=col,AverageError \
                   | tee $tmpFile \
                   | sddscombine -pipe=in $tmpFile.merged -merge -overWrite} result] {
        return -code error "Error calculating average errors: $result"
    }
    if [catch {exec sddsxref $measuredResponseFile $tmpFile -pipe=out -take=AverageError \
                   | sddsconvert -pipe -del=col,$dispColumn \
                   | sddsprocess -pipe "-redef=col,%s,%s AverageError +,select=*,exclude=BPMName" \
                   | sddsconvert -pipe=in  $tmpFile.1 -del=col,AverageError} result] {
        return -code error "Error subtracting average: $result"
    }
    if {[lsearch -exact [exec sddsquery -col $measuredResponseFile] $dispColumn] != -1 } {
        exec sddsxref $tmpFile.1 $measuredResponseFile $tmpFile.2 -take=$dispColumn
        file copy -force $tmpFile.2 $measuredResponseFile
        file delete $tmpFile.2
    } else {
        file copy -force $tmpFile.1 $measuredResponseFile
    }
    
    exec sddsconvert $averageErrorFile $tmpFile.x -keepPages=1
    exec sddsconvert $averageErrorFile $tmpFile.y -keepPages=2
    switch -exact -- $plane {
        X {set inputFile $tmpFile.x}
        Y {set inputFile $tmpFile.y}
    }
    if [catch {exec sddsxref $inputFile $tmpFile.merged -pipe=out -take=AverageError -match=BPMName \
                   -rename=col,AverageError=AverageError1 \
                   | sddsprocess -pipe "-redef=col,AverageError,AverageError AverageError1 +" \
                   | sddsconvert -pipe=in  $tmpFile.2 -del=col,AverageError1} result] {
        return -code error "Error updating average error file: $result"
    }
    switch -exact -- $plane {
        X {exec sddscombine $tmpFile.2 $tmpFile.y $averageErrorFile -overWrite}
        Y {exec sddscombine $tmpFile.x $tmpFile.2 $averageErrorFile -overWrite}
    }
    file delete $tmpFile $tmpFile.1 $tmpFile.2 $tmpFile.x $tmpFile.y $tmpFile.merged
}

#-------------------------------------------------------------------------------------------------------------------------
# This procedure calculates response matrices:
#    1. Runs acceleratorCode to calculate Response Matrix
#    2. Removes unused BPMs from the RM
#    3. Applies calibration correction to correctors (corrector tilts are in calculated matrix)
#    4. Applies gain and tilt correction to BPMs
#    5. Adds energy change to horizontal RM
#    6. Applies nonlinearity correction for BPMs

proc CalculateMatrices { args } {
    
    APSParseArguments { hrmFile vrmFile residualError nonlinLevel acceleratorCode \
                                  dispersionFile hrmMeasured vrmMeasured allElementsFile optimNonlin \
                                  coupledMatrix bpmTiltLevel useKickFiles kickFileX kickFileY currentIteration}
    
    global workDir tmpDir options twiFile dispColumn nonlinFile directRMCalc tiltCorFraction
    global solutionParamFileList verbose

    if [catch {Fit_DeleteElementsFromList -elementList [join [exec sddsquery -col $hrmMeasured] ] \
                   -deleteElements "BPMName $dispColumn"} hCorrList] {
        return -code error "Fit_DeleteElementsFromList: $hCorrList"
    }
    if [catch {Fit_DeleteElementsFromList -elementList [join [exec sddsquery -col $vrmMeasured] ] \
                   -deleteElements "BPMName $dispColumn"} vCorrList] {
        return -code error "Fit_DeleteElementsFromList: $vCorrList"
    }
    set time1 [exec date +%s]
    if [catch {CalculateResponseMatrix \
		   -acceleratorCode $acceleratorCode -directCalc $directRMCalc \
		   -latticeParamFileList $solutionParamFileList \
                   -hrmFile $hrmFile -vrmFile $vrmFile -hCorrList $hCorrList -vCorrList $vCorrList \
                   -workDir $workDir -twiFile $twiFile -coupledMatrix $coupledMatrix \
                   -fitDispersion $options(fitDispersion) -useKickFiles $useKickFiles \
		   -kickFileX $kickFileX -kickFileY $kickFileY} result] {
        return -code error "CalculateResponseMatrix: $result"
    }
    set time2 [exec date +%s]
    OutputStatusMessage "calculateResponseMatrix took [expr $time2 - $time1] sec." -logFileOnly 1
    set hYes [llength $hCorrList]
    set vYes [llength $vCorrList]
    set tmpRoot $tmpDir/[APSTmpString]-calcMatr
    set planeFlagList [list $hYes $vYes $hYes $vYes]
    set computedFileList [list $hrmFile $vrmFile $hrmFile.XRM $vrmFile.XRM]
    set measuredFileList [list $hrmMeasured $vrmMeasured $hrmMeasured $vrmMeasured]

    #------ Filtering out not used bpms by comparing with measuredFile...
    foreach planeFlag $planeFlagList computedFile $computedFileList measuredFile $measuredFileList {
        if $planeFlag {
	    if [file exists $computedFile] {
		#------ XRM (cross response) might not exist
		exec sddsconvert $computedFile -pipe=out -delete=column,s \
		    | sddsselect -pipe=in $measuredFile $tmpRoot -match=BPMName
		if $verbose {
		    if {[exec sddscombine $tmpRoot -pipe=out -merge | sdds2stream -pipe=in -rows=bare] == 0} {
			OutputStatusMessage \
			    "Error: zero rows after filtering (computedFile: $computedFile; measuredFile: $measuredFile)" \
			    -logFileOnly 1
			return -code error "Error: zero rows after filtering."
		    }
		}
		file copy -force $tmpRoot $computedFile
	    }
	}
    }
    file delete $tmpRoot

    set time1 [exec date +%s]
    #------ Corrector calibration is now in parameter files and is included automatically in the RM calculation
    #------ Here we do dispersion only:
    if $options(useCorrs) {
	if [catch {ApplyCorrectorCalibrationCorrection -hrmFile $hrmFile -vrmFile $vrmFile -hrmXFile $hrmFile.XRM \
		       -vrmXFile $vrmFile.XRM -hYes $hYes -vYes $vYes -allElementsFile $allElementsFile -dispersionOnly 1} result] {
	    return -code error "ApplyCorrectorCalibrationCorrection: $result"
	}
    }
    set time2 [exec date +%s]
    OutputStatusMessage "Corrector calibration took [expr $time2 - $time1] sec." -logFileOnly 1

    #------ This procedure find bpms tilts on a bpm-by-bpm basis without Inverse RMD (adds tilts to RMD results if they exist):
    set time1 [exec date +%s]
    if {$residualError < $bpmTiltLevel} {
	global bpmTiltIterCounter corFraction iterations
	switch $bpmTiltIterCounter {
	    0 { set bpmTiltFraction [expr $corFraction * $tiltCorFraction * 0.1] }
	    1 { set bpmTiltFraction [expr $corFraction * $tiltCorFraction * 0.2] }
	    2 { set bpmTiltFraction [expr $corFraction * $tiltCorFraction * 0.4] }
	    default { set bpmTiltFraction [expr $corFraction * $tiltCorFraction] }
	}
        if {$currentIteration != -1} {
	    if [catch {CalculateSingleBPMTilts -hrmXFile $hrmFile.XRM -vrmXFile $vrmFile.XRM \
			   -allElementsFile $allElementsFile -coupledMatrix $coupledMatrix -corFraction $bpmTiltFraction} result] { 
		return -code error "CalculateSingleBPMTilts: $result"
	    }
	    incr bpmTiltIterCounter
	}
    }
    set time2 [exec date +%s]
    OutputStatusMessage "Bpm tilt calculation took [expr $time2 - $time1] sec." -logFileOnly 1

    set time1 [exec date +%s]
    if [catch {ApplyBPMCorrections -hrmFile $hrmFile -hrmXFile $hrmFile.XRM -vrmFile $vrmFile -vrmXFile $vrmFile.XRM \
		   -allElementsFile $allElementsFile -hYes $hYes -vYes $vYes -coupledMatrix $coupledMatrix \
		   -residualError $residualError -bpmTiltLevel $bpmTiltLevel} result] {
	return -code error "ApplyBPMCorrections: $result"
    }
    set time2 [exec date +%s]
    OutputStatusMessage "Bpm correction took [expr $time2 - $time1] sec." -logFileOnly 1
    
    if $options(useEnergy) {
        if [catch {AddEnergy2ResponseMatrix -responseFile $hrmFile -allElementsFile $allElementsFile \
                       -dispersionFile $dispersionFile} result] {
            return -code error "AddEnergy2ResponseMatrix: $result"
        }
    }
    #------ Nonlin correction is not applied to cross-response
    if {$residualError < $nonlinLevel} {
	set time1 [exec date +%s]
        set nonlinFileList [list $nonlinFile.X $nonlinFile.Y]
	set planeFlagList [list $hYes $vYes]
	set computedFileList [list $hrmFile $vrmFile]
	set measuredFileList [list $hrmMeasured $vrmMeasured]
	set pageList [list 1 2]
        foreach planeFlag $planeFlagList computedFile $computedFileList measuredFile $measuredFileList \
            nonlFile $nonlinFileList usePage $pageList {
		if [file exists $computedFile] { 
		    if [catch {ApplyNonlinCorrection -computedFile $computedFile -measuredFile $measuredFile \
				   -nonlinFile $nonlFile -coupledMatrix $coupledMatrix -usePage $usePage} result] {
			return -code error "ApplyNonlinCorrection ($nonlFile): $result"
		    }
		}
            }
	set time2 [exec date +%s]
	OutputStatusMessage "Nonlin correction took [expr $time2 - $time1] sec." -logFileOnly 1
    }
    set nuList [exec sdds2stream $twiFile -para=nux,nuy]
#    exec sddsprocess $hrmFile -nowarning -def=para,nux,[lindex $nuList 0] -def=para,nuy,[lindex $nuList 1]
#    exec sddsprocess $vrmFile -nowarning -def=para,nux,[lindex $nuList 0] -def=para,nuy,[lindex $nuList 1]
    file delete ${hrmFile}~ ${vrmFile}~
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateResponseMatrix { args } {    
    global locoBinDir verbose bRho

    set getSDDSLattice 0
    set getBetaFunctions 0
    set lteFile ""
    set beamlineName ""
    set latticeParamFileList ""
    set optimFile ""
    set sddsLatticeFile ""
    set twiFile ""
    set workDir ""
    set hrmFile ""
    set vrmFile ""
    set hCorrList ""
    set vCorrList ""
    set useQsub ""
#    set bRho ""
    set dispFitWeight ""
    set directCalc ""
    set kickX ""
    set kickY ""
    set fixedOrbitLength ""
    set fitDispersion ""
    set dispColumn ""
    set optimNonlin ""
    set coupledMatrix ""
    set closedOrbit ""
    set useDoubleOrbits ""
    set acceleratorCode ""
    set locoSearchString ""
    set numberOfQsubTasks ""
    set waitTime ""
    set waitInterval ""
    set qsubCommand ""
    set queueSystemName ""
    set qsubRespProcCommand ""
    set submissionPause ""
    set twissRefFile ""
    set twissRefElement ""
    set malignElement ""
    set useKickFiles 0
    set kickFileX ""
    set kickFileY ""

    APSParseArguments { getSDDSLattice getBetaFunctions lteFile beamlineName latticeParamFileList optimFile \
			    sddsLatticeFile twiFile workDir \
                            hrmFile vrmFile hCorrList vCorrList acceleratorCode closedOrbit useDoubleOrbits \
                            useQsub bRho dispFitWeight directCalc kickX kickY numberOfQsubTasks \
                            fixedOrbitLength fitDispersion dispColumn optimNonlin coupledMatrix \
			    waitTime waitInterval qsubCommand queueSystemName qsubRespProcCommand submissionPause \
			    twissRefFile twissRefElement malignElement useKickFiles kickFileX kickFileY}

    if $getSDDSLattice {
        if [catch {exec $locoBinDir/calculateResponseMatrix -runMode getSDDSLattice -acceleratorCode $acceleratorCode \
                       -workDir $workDir -lteFile $lteFile -beamlineName $beamlineName -bRho $bRho \
		       -latticeParamFileList $latticeParamFileList -optimFile $optimFile \
		       -sddsLatticeFile $sddsLatticeFile >@ stdout} result] {
            return -code error "calculateResponseMatrix (mode getSDDSLattice): $result"
        }
        return 0
    }
    if $getBetaFunctions {
	if ![string length $closedOrbit] {set closedOrbit 1}
        if [catch {exec $locoBinDir/calculateResponseMatrix -runMode getBetaFunctions -acceleratorCode $acceleratorCode \
                       -workDir $workDir -lteFile $lteFile -beamlineName $beamlineName -bRho $bRho \
		       -latticeParamFileList $latticeParamFileList -optimFile $optimFile \
		       -twiFile $twiFile \
		       -closedOrbit $closedOrbit -twissRefFile $twissRefFile \
		       -twissRefElement $twissRefElement -malignElement $malignElement >@ stdout} result] {
            return -code error "calculateResponseMatrix (mode getBetaFunctions): $result"
        }
        return 0
    }
    
    #------ Main call.
    #------ List of variables which has the same name as arguments of the calculateResponseMatrix:
    set globalVarList [list bRho dispFitWeight fixedOrbitLength dispColumn optimNonlin closedOrbit \
			   coupledMatrix locoSearchString \
			   lteFile beamlineName latticeParamFileList optimFile useDoubleOrbits \
			   twissRefFile twissRefElement malignElement]
    foreach var $globalVarList {
        if ![string length [set $var]] {
            unset $var
            global $var
        }
    }
    set possibleEmptyVars [list twissRefFile twissRefElement malignElement kickFileX kickFileY]
    foreach var $possibleEmptyVars {
	if ![string length [set $var]] {set $var dummy}
    }
    #------ List of the variables, which are different from the arguments of the calculateResponseMatrix:
    set optionsVariablesList [list {useQsubShort useQsub} {qsubCommandShort qsubCommand} {waitTimeShort waitTime} \
				  {numberOfQsubTasksShort numberOfQsubTasks} {waitIntervalShort waitInterval} \
				  {directRMCalc directCalc} {thetaKickX kickX} {thetaKickY kickY} \
				  {submissionPauseShort submissionPause} {queueSystemNameShort queueSystemName} \
				  {qsubRespProcCommandShort qsubRespProcCommand}]
    foreach optionList $optionsVariablesList {
	set globalVar [lindex $optionList 0]
	set optionName [lindex $optionList 1]
        if ![string length [set $optionName]] {
            global $globalVar
            set $optionName [set $globalVar]
        }
    }
    if {$useQsub && ![string length $numberOfQsubTasks]} {
	return -code error "Variable numberOfQsubTasks in not defined."
    }
    if ![string length $fitDispersion] {
        global options
        set fitDispersion $options(fitDispersion)
    }
    #------ calculateResponseMatrix runs appropriate accelerator code to calculate response matrix
    set runCommand "exec $locoBinDir/calculateResponseMatrix \
                   -runMode getResponseMatrix \
		   -acceleratorCode $acceleratorCode \
                   -lteFile $lteFile \
		   -beamlineName $beamlineName \
		   -latticeParamFileList \"$latticeParamFileList\" \
		   -optimFile $optimFile \
                   -hrmFile $hrmFile \
                   -vrmFile $vrmFile \
                   -hCorrList \"$hCorrList\" \
                   -vCorrList \"$vCorrList\" \
                   -workDir $workDir \
                   -useQsub $useQsub \
                   -bRho $bRho \
                   -twiFile $twiFile \
                   -dispFitWeight $dispFitWeight \
                   -directCalc $directCalc \
                   -kickX $kickX \
                   -kickY $kickY \
	           -useKickFiles $useKickFiles \
	           -kickFileX $kickFileX \
	           -kickFileY $kickFileY \
                   -fixedOrbitLength $fixedOrbitLength \
                   -fitDispersion $fitDispersion \
                   -dispColumn $dispColumn \
                   -optimNonlin $optimNonlin \
                   -coupledMatrix $coupledMatrix \
		   -closedOrbit $closedOrbit \
		   -twissRefFile $twissRefFile \
		   -twissRefElement $twissRefElement \
		   -malignElement $malignElement \
                   -useDoubleOrbits $useDoubleOrbits \
		   -locoSearchString $locoSearchString \
		   -numberOfQsubTasks $numberOfQsubTasks \
		   -waitTime $waitTime \
		   -waitInterval $waitInterval \
		   -qsubCommand \"[ProtectDollarSigns $qsubCommand]\" \
		   -queueSystemName $queueSystemName \
		   -submissionPause $submissionPause \
		   -qsubRespProcCommand \"[ProtectDollarSigns $qsubRespProcCommand]\"\
                   -verbose $verbose"
#    if $verbose {OutputStatusMessage $runCommand -logFileOnly 1}
    OutputStatusMessage $runCommand -logFileOnly 1
    if [catch {eval $runCommand  >@ stdout} result] {
        return -code error "calculateResponseMatrix (mode getResponseMatrix): $result"
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc ApplyCorrectorCalibrationCorrection {args } {
    global tmpDir dispColumn options
    APSParseArguments {hrmFile vrmFile hrmXFile vrmXFile allElementsFile hYes vYes dispersionOnly}

    set tmpRoot $tmpDir/[APSTmpString]-calibCorr
    if $dispersionOnly {
	if $options(fitDispersion) {
	    set dispGain [exec sddsprocess $allElementsFile.hCorr -pipe=out -match=col,ElementName=$dispColumn \
			      | sdds2stream -pipe=in -col=ParameterValue]
	    exec sddsprocess $hrmFile $tmpRoot "-redef=col,$dispColumn,$dispColumn 1 $dispGain + *"
	    file copy -force $tmpRoot $hrmFile
	    file delete $tmpRoot
	}
    } else {
	set hList [list $hrmFile $hrmXFile]
	set vList [list $vrmFile $vrmXFile]
	foreach planeFlag [list $hYes $vYes] fileList [list $hList $vList] \
	    corrFile [list $allElementsFile.hCorr $allElementsFile.vCorr] {
		if {$planeFlag && [exec sdds2stream $corrFile -rows=bare]} {
		    foreach calcFile $fileList {
			if [file exists $calcFile] {
			    set rows [lindex [exec sdds2stream $calcFile -rows=bare] 0]
			    if [catch {exec sddscombine $calcFile -pipe=out -merge \
					   | sddstranspose -pipe -oldColumnNames=CorrectorNameValue -newColumnNames=BPMName \
					   | sddsxref -pipe $corrFile -fillIn -take=ParameterValue -match=CorrectorNameValue=ElementName \
					   | sddsprocess -pipe "-redef=col,%s,%s 1 ParameterValue + *,select=*,exclude=*Value" \
					   | sddsconvert -pipe -del=col,ParameterValue -del=para,* \
					   | sddstranspose -pipe -oldColumnNames=BPMName -newColumnNames=CorrectorNameValue \
					   | sddsbreak -pipe -rowlimit=$rows \
					   | sddsxref -pipe=in $calcFile $tmpRoot.out -leave=* -transfer=para,* } result] {
				OutputStatusMessage \
				    "Error applying calibration correction (calcFile: $calcFile; corrFile: $corrFile): $result" \
				    -logFileOnly 1
				return -code error "Error applying calibration correction: $result"
			    }
			    file copy -force $tmpRoot.out $calcFile
			}
		    }
		}
	    }
	file delete $tmpRoot.out
    }
}

#-------------------------------------------------------------------------------------------------------------------------
#------ Xbpm = ( Xorb * Cos A + Yorb * Sin A) * (1 + Gx)
#------ Ybpm = (-Xorb * Sin A + Yorb * Cos A) * (1 + Gy)
#------ where Zbpm  -  bpm reading
#------       Zorb  -  real orbit
#------          A  -  tilt
#------         Gz  -  bpm gain error
#---
proc ApplyBPMCorrections { args } {

    global tmpDir options
    APSParseArguments { hrmFile hrmXFile vrmFile vrmXFile allElementsFile hYes vYes coupledMatrix \
			    residualError bpmTiltLevel }
    
    set tmpRoot $tmpDir/[APSTmpString]-bpmCorrect

    set tmpFileList ""
    set xbpmTiltFile $allElementsFile.xBpmTilt
    set ybpmTiltFile $allElementsFile.yBpmTilt
    set xbpmGainFile $allElementsFile.xBpm
    set ybpmGainFile $allElementsFile.yBpm

    set doTiltCorrection 1
    if {![file exists $xbpmTiltFile] || ![file exists $ybpmTiltFile] || ![file exists $hrmXFile] \
	    || ![file exists $vrmXFile]} {
	set doTiltCorrection 0
    }
    if {!$options(useBPMsTilt) && ($residualError > $bpmTiltLevel)} {
	set doTiltCorrection 0
    }

    #------ Tilt correction
    if $doTiltCorrection {
	set xbpms [lindex [exec sdds2stream $hrmFile -rows=bare] 0]
	exec sddsprocess $xbpmTiltFile $tmpRoot.sin1 "-redef=col,SinValue,ParameterValue sin"
	exec sddsprocess $ybpmTiltFile $tmpRoot.sin2 "-redef=col,SinValue,ParameterValue sin -1 *"
	exec sddscombine $xbpmTiltFile $ybpmTiltFile -pipe=out -merge \
	    | sddsprocess -pipe=in $tmpRoot.directVector "-redef=col,CosValue,ParameterValue cos"
	exec sddscombine $tmpRoot.sin1 $tmpRoot.sin2 $tmpRoot.crossVector -merge -overWrite
	lappend tmpFileList $tmpRoot.sin1 $tmpRoot.sin2 $tmpRoot.directVector $tmpRoot.crossVector
	
	foreach rmFile [list $hrmFile $vrmFile $hrmXFile $vrmXFile] \
	    vectorFile [list $tmpRoot.directVector $tmpRoot.directVector $tmpRoot.crossVector $tmpRoot.crossVector] \
	    columnName [list CosValue CosValue SinValue SinValue] \
	    extName [list hDirect vDirect hCross vCross] {
		if [catch {exec sddscombine $rmFile -pipe=out -merge \
			       | sddsxref -pipe $vectorFile -nowarning -fillIn -take=$columnName -match=BPMName=ElementName \
			       | sddsconvert -pipe -rename=col,BPMName=BPMNameValue \
			       | sddsprocess -pipe "-redef=col,%s,%s $columnName *,select=*,exclude=*Value" \
			       | sddsconvert -pipe=in $tmpRoot.$extName -rename=col,BPMNameValue=BPMName \
			       -del=col,$columnName} result] {
		    return -code error "Error applying bpm tilts: $result"
		}
		lappend tmpFileList $tmpRoot.$extName
	    }
	exec sddsmatrixop $tmpRoot.hDirect -pipe=out -push=$tmpRoot.hCross -add \
	    | sddsprocess -pipe "-redef=col,Break,i_row $xbpms < pop pop ? 0 : 1 \$" \
	    | sddsbreak -pipe -change=Break \
	    | sddsconvert -pipe -del=col,Break \
	    | sddsxref -pipe=in $hrmFile $tmpRoot.hResult -take=BPMName -transfer=para,*
	exec sddsmatrixop $tmpRoot.vDirect -pipe=out -push=$tmpRoot.vCross -add \
	    | sddsprocess -pipe "-redef=col,Break,i_row $xbpms < pop pop ? 0 : 1 \$" \
	    | sddsbreak -pipe -change=Break \
	    | sddsconvert -pipe -del=col,Break \
	    | sddsxref -pipe=in $vrmFile $tmpRoot.vResult -take=BPMName -transfer=para,*
	lappend tmpFileList $tmpRoot.hResult $tmpRoot.vResult
	file copy -force $tmpRoot.hResult $hrmFile
	file copy -force $tmpRoot.vResult $vrmFile
    }

    #------ Gain correction
    set planeFlagList [list $hYes $vYes]
    set rmFileList [list $hrmFile $vrmFile]
    set planeList [list X Y]
    set bpmFile $tmpRoot.bpmGain
    foreach planeFlag $planeFlagList rmFile $rmFileList plane $planeList {
        if $planeFlag {
	    switch -regexp -- $coupledMatrix {
		0|1 { 
		    switch -exact -- $plane {
			X {set bpmFile $xbpmGainFile}
			Y {set bpmFile $ybpmGainFile}
		    }
		}
		2 {exec sddscombine $allElementsFile.xBpm $allElementsFile.yBpm $bpmFile -overWrite; lappend tmpFileList $bpmFile}
	    }
	    if [exec sddscombine $bpmFile -pipe=out -merge | sdds2stream -pipe=in -rows=bare] {
		if [catch {exec sddsconvert $rmFile -pipe=out -rename=col,BPMName=BPMNameValue \
			       | sddsxref -nowarning -pipe $bpmFile -fillIn -take=ParameterValue -match=BPMNameValue=ElementName \
			       | sddsprocess -pipe "-redef=col,%s,%s 1 ParameterValue + *,select=*,exclude=*Value" \
			       | sddsconvert -pipe=in $tmpRoot.gainResult -del=col,ParameterValue \
			       -rename=col,BPMNameValue=BPMName} result] {
		    return -code error "BPM gain correction error: $result"
		}
		file copy -force $tmpRoot.gainResult $rmFile
		lappend tmpFileList $tmpRoot.gainResult 
	    }
        }
    }
    catch {eval file delete $tmpFileList}
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateSingleBPMTilts { args } {
    global workDir tmpDir hrmDifference vrmDifference
    APSParseArguments {hrmXFile vrmXFile allElementsFile coupledMatrix corFraction}
    
    #------ To find tilts of X bpms use:
    #------ set differenceFile RMvdifference.flvrm
    #------ set computedXRM computed.flvrm.XRM
    #------ set pageNumber 1
    #------ To find tilts of Y bpms use:
    #------ set differenceFile RMhdifference.flhrm
    #------ set computedXRM computed.flhrm.XRM
    #------ set pageNumber 2

    set tiltGain $corFraction
    set tmpRoot $tmpDir/[APSTmpString]-singleTilts
    set deleteFiles ""
    if {$coupledMatrix == 2} {
	exec sddsprocess $allElementsFile.xBpmTilt -redef=para,SinSign,1 -nowarning
	exec sddsprocess $allElementsFile.yBpmTilt -redef=para,SinSign,-1 -nowarning
	lappend deleteFiles $allElementsFile.xBpmTilt~ $allElementsFile.yBpmTilt~

	#------ Find BPM tilts:
	foreach differenceFile [list $vrmDifference $hrmDifference] computedXRM [list $vrmXFile $hrmXFile] \
	    pageNumber [list 1 2] ext [list xBpmTilt yBpmTilt] sign [list 1 -1] {
		set bpmTiltFile $allElementsFile.$ext
		set outputFile $workDir/bpmTilts.$ext
		#------ Find BPM tilts:
		if [catch {GetBPMTilts -differenceFile $differenceFile -computedXRM $computedXRM \
			       -outputFile $outputFile -pageNumber $pageNumber} result] {
		    return -code error "GetBPMTilts: $result"
		}
		set stdev$ext [exec sddsprocess $outputFile -pipe=out -process=Tilt,standarddeviation,StDev \
			       | sdds2stream -pipe=in -para=StDev]
		#------ Store bpm tilts in allElementsFile:
		exec sddsxref $bpmTiltFile $outputFile -pipe=out -nowarning -take=Tilt \
		    -match=ElementName=BPMName -fillIn \
		    | sddsprocess -pipe "-redef=col,ParameterValue,ParameterValue Tilt $sign * $tiltGain * +" \
		    | sddsconvert -pipe=in $tmpRoot.$ext -del=col,Tilt
		file copy -force $tmpRoot.$ext $bpmTiltFile
		lappend deleteFiles $tmpRoot.$ext
		if 0 {
		    set counter 0
		    while {[file exists $outputFile.plot] == 1} {
			if [file exists $outputFile.plot.$counter] {
			    incr counter
			} else {
			    file rename -force $outputFile.plot $outputFile.plot.$counter
			    file rename -force $outputFile $outputFile.$counter
			}
		    }
		}
	    }
	OutputStatusMessage "BPM tilts rms: [format %10.2e $stdevxBpmTilt] -- [format %10.2e $stdevyBpmTilt]"
    }
    catch {eval file delete $deleteFiles}
}

#-------------------------------------------------------------------------------------------------------------------------

proc GetBPMTilts { args } {
    global tmpDir
    APSParseArguments {differenceFile computedXRM outputFile pageNumber}

    set tmpRoot $tmpDir/[APSTmpString]-bpmTilts
    set steps 40
    set alpha1 -0.001
    set alpha2 0.001
    set multiplier 4
    set counterLimit 10

    #------ Keeping one page:
    exec sddsconvert $differenceFile $differenceFile.page -keep=$pageNumber
    exec sddsconvert $computedXRM $computedXRM.page -keep=$pageNumber

    #------ File for BPM order:
    exec sddsprocess $differenceFile.page $tmpRoot.order -def=col,BPMOrder,i_row
    lappend deleteFiles $tmpRoot.order
    
    #------ Combining files:
    set fileList $differenceFile.page
    for {set I 0} {$I <= $steps} {incr I} {
	lappend fileList $computedXRM.page
    }
    eval exec sddscombine $fileList $tmpRoot.big -overWrite
    lappend deleteFiles $tmpRoot.big
    
    #------ Loop over fit limits:
    set rows [exec sdds2stream $computedXRM.page -rows=bare]
    file copy -force $tmpRoot.big $tmpRoot.big.outliers
    lappend deleteFiles $tmpRoot.big.outliers
    set counter 0
    while {$rows != 0} {
	if [catch {FindBPMTilts -inputFile $tmpRoot.big.outliers -outputFile $tmpRoot.tilts -plotFile $tmpRoot.plot \
		       -steps $steps -a1 $alpha1 -a2 $alpha2} result] {
	    return -code error "FindBPMTilts(differenceFile: $differenceFile; computedXRM: $computedXRM; pageNumber: $pageNumber): $result"
	}
	lappend deleteFiles $tmpRoot.tilts $tmpRoot.plot
	if !$counter {
	    file copy -force $tmpRoot.tilts $outputFile
	    #------ For plots: keep only those pages that are within present limits:
	    exec sddsprocess $tmpRoot.plot $outputFile.plot -filter=para,Tilt,[expr $alpha1 + 1e-10],[expr $alpha2 - 1e-10]
	} else {
	    #------ Substitute rows for expanded values:
	    exec sddsxref $outputFile $tmpRoot.tilts -pipe=out -take=Tilt -match=BPMName -fillin \
		-rename=col,Tilt=Tilt1 -nowarning \
		| sddsprocess -pipe "-redef=col,Tilt,Tilt1 0 == ? Tilt : Tilt1 \$" \
		| sddsconvert -pipe=in $tmpRoot.copy -del=col,Tilt1
	    file copy -force $tmpRoot.copy $outputFile
	    lappend deleteFiles $tmpRoot.copy
	    #------ For plots: combine with previous iteration, then keep only those pages that are within present limits:
	    exec sddscombine $outputFile.plot $tmpRoot.plot -pipe=out \
		| sddsprocess -pipe=in $outputFile.plot.copy -filter=para,Tilt,[expr $alpha1 + 1e-10],[expr $alpha2 - 1e-10]
	    file copy -force $outputFile.plot.copy $outputFile.plot
	    lappend deleteFiles $outputFile.plot.copy
	}
	#------ File with rows that are outside present limits:
	exec sddsprocess $tmpRoot.tilts $tmpRoot.outliers -nowarning "-filter=col,Tilt,-1e10,$alpha1,Tilt,$alpha2,1e10,|"
	lappend deleteFiles $tmpRoot.outliers
	#------ For next iteration keep only pages that are outside present limits:
	exec sddsselect $tmpRoot.big $tmpRoot.outliers $tmpRoot.big.outliers -match=BPMName -reuse=page
	set rows [exec sdds2stream $tmpRoot.outliers -rows=bare]
	set alpha1 [expr $alpha1 * $multiplier]
	set alpha2 [expr $alpha2 * $multiplier]
	incr counter
	if {$counter > $counterLimit} {
	    exec sddsprocess $outputFile "-redef=col,Tilt, Tilt $alpha1 < Tilt $alpha2 > || ? 0 : Tilt \$" -nowarning
	    lappend deleteFiles ${outputFile}~
	    OutputStatusMessage "Warning: GetBPMTilts: Counter exceeded maximum value. Outlying bpm tilts are set to 0."
	    break
	}
    }
    
    #------ Sorting output file:
    exec sddsxref $outputFile $tmpRoot.order -pipe=out -take=BPMOrder -match=BPMName \
	| sddssort -pipe=in $outputFile.copy -col=BPMOrder
    file copy -force $outputFile.copy $outputFile
    lappend deleteFiles $outputFile.copy
    #------ Sorting output plot file:
    exec sddscollapse $outputFile.plot -pipe=out \
	| sddsxref -pipe $tmpRoot.order -take=BPMOrder -match=BPMName \
	| sddsbreak -pipe -rowLimit=1 \
	| sddsprocess -pipe=in $tmpRoot.order1 -process=BPMOrder,first,BPMOrder
    exec sddsxref $outputFile.plot $tmpRoot.order1 -pipe=out -leave=* -transfer=para,BPMOrder \
	| sddssort -pipe=in $outputFile.plot.copy -para=BPMOrder
    file copy -force $outputFile.plot.copy $outputFile.plot
    eval file delete $deleteFiles $outputFile.plot.copy $tmpRoot.order1
    
}

#------------------------------------------------------------------------------------------------------------------------------

proc FindBPMTilts {args} {
    set plotFile ""
    set deletePlotFile 0
    set argsCopy $args
    APSParseArguments {inputFile outputFile plotFile steps a1 a2}
    if ![string length $plotFile] {
	set plotFile tmpPlotFile
	set deletePlotFile 1
    }
    if [catch {exec sddsprocess $inputFile -pipe=out "-redef=para,Alpha,i_page 1 == ? 1 : $a1 $a2 $a1 - $steps / i_page 2 - * + \$" \
		   | sddstranspose -pipe -newcolumn=BPMName -oldcolumn=Corrector \
		   | sddsprocess -pipe "-redef=col,%s,%s Alpha *,select=*,exclude=Corrector" \
		   | sddschanges -pipe -change=exclude=Corrector,* -copy=Corrector \
		   | sddstranspose -pipe -newcolumn=Corrector -oldcolumn=BPMName \
		   | sddsrowstats -pipe -standardDeviation=StDev,* \
		   | sddsprocess -pipe "-reedit=col,BPMName,8d" \
		   | sddsconvert -pipe -retain=col,BPMName,StDev \
		   | sddsprocess -pipe -def=col,Tilt,Alpha \
		   | sddscombine -pipe -merge \
		   | sddssort -pipe -col=BPMName \
		   | sddsbreak -pipe -change=BPMName \
		   | sddsprocess -pipe -process=BPMName,first,BPMName \
		   | sddssort -pipe -col=Tilt \
		   | sddsprocess -pipe -process=StDev,min,Tilt,position,functionOf=Tilt \
		   | tee $plotFile \
		   | sddscollapse -pipe=in $outputFile} result] {
	return -code error "$result -- calling argument: $argsCopy"
    }
    if $deletePlotFile { file delete tmpPlotFile }
}

#------------------------------------------------------------------------------------------------------------------------------

proc AddEnergy2ResponseMatrix { args } {
    
    global tmpDir options dispColumn
    APSParseArguments { responseFile dispersionFile allElementsFile }
    
    set tmpRoot $tmpDir/[APSTmpString]-addEnergy
    set energyFile $tmpRoot.energy
    lappend deleteFiles $energyFile
    exec sddsconvert $allElementsFile.Energy $energyFile -retain=col,ElementName,ParameterValue
    set twoPageFile [exec sdds2stream $responseFile -npages=bare]
    if {$twoPageFile == 2} {
        exec sddsprocess $responseFile -pipe=out -define=col,PageNumber,i_page \
            | sddsconvert -pipe -retain=col,PageNumber \
            | sddscombine -pipe=in $tmpRoot.pageNumber -merge -overWrite
        exec sddscombine $dispersionFile $tmpRoot.disp -merge
        exec sddscombine $responseFile $tmpRoot.response -merge
	lappend deleteFiles $tmpRoot.pageNumber $tmpRoot.disp $tmpRoot.response
    } else {
        exec sddsprocess $dispersionFile $tmpRoot.disp -match=para,Plane=X
        file copy -force $responseFile $tmpRoot.response
	lappend deleteFiles $tmpRoot.disp $tmpRoot.response
    }
    #------ Add zero row at the end of energy file to match the dimensions of the response file with dispersion:
    if {[lsearch -exact [join [exec sddsquery $responseFile -col] ] $dispColumn] != -1} {
	exec sddsmakedataset $tmpRoot.lastrow -col=ElementName,type=string -data=$dispColumn \
	    -col=ParameterValue,type=double -data=0
	exec sddscombine $energyFile $tmpRoot.lastrow $tmpRoot.energy1 -merge
	set energyFile $tmpRoot.energy1
	lappend deleteFiles $tmpRoot.lastrow $tmpRoot.energy1
    }
    if [catch {exec sddsmatrixop $tmpRoot.disp -pipe=out -push=$energyFile -transpose -multiply -push=$tmpRoot.response -add \
		   -columnNames=file=$allElementsFile.hCorr,columnName=ElementName \
		   | sddsxref -pipe=in $tmpRoot.response $tmpRoot -take=BPMName -transfer=para,nux,nuy,Kick} result] {
	return -code error "Error adding matrices ($responseFile, $tmpRoot.disp, $energyFile, and $tmpRoot.response): $result"
    }
    lappend deleteFiles $tmpRoot
    if {$twoPageFile == 2} {
        exec sddsxref $tmpRoot $tmpRoot.pageNumber -pipe=out -take=PageNumber \
            | sddsbreak -pipe -changeOf=PageNumber \
            | sddsconvert -pipe=in $responseFile -del=col,PageNumber
    } else {
        file copy -force $tmpRoot $responseFile
    }
    eval file delete $deleteFiles
}

#-------------------------------------------------------------------------------------------------------------------------

proc ApplyNonlinCorrection { args } {

    APSParseArguments { computedFile measuredFile nonlinFile coupledMatrix usePage }

    global tmpDir dispColumn
    set tmpRoot $tmpDir/[APSTmpString]-nonlin

    #------ Previously useSlope was 1, but that moves part of BPM gain to nonlinearity file.
    global useIntercept useSlope useCubic
    if ![info exists useIntercept] {set useIntercept 0}
    if ![info exists useSlope] {set useSlope 0}
    if ![info exists useCubic] {set useCubic 1}
    OutputStatusMessage "NonlinCorrection: $useIntercept -- $useSlope -- $useCubic" -logFileOnly 1

    set deleteFiles ""

    #------ We don't store previous nonlinear terms - calculate full on each iteration.

    #------ Don't understand cubic term in X-Y and Y-X responses, therefore don't use nonlinear correction for
    #------ corresponsing pages.

    #------ Calculate BPM nonlinearities:
    if {$coupledMatrix == 2} {
	set page $usePage
	if {$page == 1} {set otherPage 2} else {set otherPage 1}
	exec sddsconvert $computedFile $tmpRoot.comp -keepPage=$page
	exec sddsconvert $measuredFile $tmpRoot.meas -keepPage=$page
	exec sddsconvert $nonlinFile $tmpRoot.nonlin.$otherPage -keepPage=$otherPage
	if [catch {GetBPMNonlinearity -computedResponseFile $tmpRoot.comp -measuredResponseFile $tmpRoot.meas \
		       -outputFile $tmpRoot.nonlin.$page} result] {
	    return -code error "GetBPMNonlinearity: $result"
	}
	lappend deleteFiles $tmpRoot.comp $tmpRoot.meas $tmpRoot.nonlin.$page $tmpRoot.nonlin.$otherPage
        exec sddscombine $tmpRoot.nonlin.1 $tmpRoot.nonlin.2 $tmpRoot.pfit -overWrite
    } else {
        if [catch {GetBPMNonlinearity -computedResponseFile $computedFile -measuredResponseFile $measuredFile \
                       -outputFile $tmpRoot.pfit} result] {
            return -code error "GetBPMNonlinearity: $result"
        }
    }
    lappend deleteFiles $tmpRoot.pfit
    exec sddsprocess $tmpRoot.pfit $nonlinFile \
	"-redef=col,pfitIntercept,pfitIntercept $useIntercept *" \
	"-redef=col,pfitSlope,pfitSlope $useSlope *" \
	"-redef=col,pfitCubic,pfitCubic $useCubic *"

    #------ Subtract nonlinear terms from the matrix:

    if {[lsearch -exact [exec sddsquery -col $computedFile] $dispColumn] != -1 } {
        exec sddsconvert $computedFile $tmpRoot.dispVector -retain=col,BPMName,$dispColumn
        exec sddsconvert $computedFile $tmpRoot.noDisp -del=col,$dispColumn
        set addDisp 1
	lappend deleteFiles $tmpRoot.dispVector $tmpRoot.noDisp
    } else {
        file copy -force $computedFile $tmpRoot.noDisp
        set addDisp 0
	lappend deleteFiles $tmpRoot.noDisp
    }
    exec sddsxref $tmpRoot.noDisp $nonlinFile -pipe=out -take=pfitIntercept,pfitSlope,pfitCubic -match=BPMName \
        | sddsconvert -pipe -del=col,BPMName \
        | sddsprocess -pipe \
        "-redef=col,%s,%s pfitIntercept $useIntercept * + %s pfitSlope $useSlope * * + %s 3 pow pfitCubic $useCubic * * +,select=*,exclude=pfit*" \
        | sddsxref -pipe $computedFile -take=BPMName \
        | sddsconvert -pipe=in $tmpRoot -del=col,pfit*
    lappend deleteFiles $tmpRoot
    if $addDisp {
        exec sddsxref $tmpRoot $tmpRoot.dispVector $computedFile -take=$dispColumn
    } else {
        file copy -force $tmpRoot $computedFile
    }
    eval file delete $deleteFiles
}

#-------------------------------------------------------------------------------------------------------------------------

proc GetBPMNonlinearity { args } {

    global tmpDir dispColumn
    APSParseArguments {measuredResponseFile computedResponseFile outputFile}

    set tmpRoot $tmpDir/[APSTmpString]-nonlin
    set measuredFile $tmpRoot.measured
    set computedFile $tmpRoot.computed
    if {[lsearch -exact [exec sddsquery -col $measuredResponseFile] $dispColumn] != -1 } {
        exec sddsconvert $measuredResponseFile $measuredFile -del=col,$dispColumn
        exec sddsconvert $computedResponseFile $computedFile -del=col,$dispColumn
    } else {
        file copy -force $measuredResponseFile $measuredFile
        file copy -force $computedResponseFile $computedFile
    }
    exec sddscollect $computedFile $tmpRoot.collect -nowarning -collect=match=*,exclude=BPMName,column=Orbit,edit=
    exec sddsmakedataset $tmpRoot.col -col=CoefficientName,type=string -data=[join [list pfitIntercept pfitSlope pfitCubic] ,]
    if [catch {exec sddschanges $measuredFile -pipe=out -base=$computedFile -copy=BPMName -changesIn=exclude=BPMName,* \
		   | sddscollect -pipe -nowarning -collect=prefix=ChangeIn \
		   | sddsxref -pipe $tmpRoot.collect -take=Orbit -match=Rootname \
		   | sddsconvert -pipe -rename=col,ChangeIn=Error \
		   | tee $tmpRoot.errors \
		   | sddspfit -pipe -col=Orbit,Error -orders=0,1,3 -generate \
		   | sddsxref -pipe $tmpRoot.errors -transfer=para,BPMName -rename=para,BPMName=BPMNameParam \
		   | sddsconvert -pipe -del=col,* \
		   | sddsarray2column -pipe -convert=Coefficient \
		   | sddsxref -pipe $tmpRoot.col -take=CoefficientName -reuse=page \
		   | sddstranspose -pipe -newColumnName=CoefficientName \
		   | sddsprocess -pipe -print=col,BPMName,%s,BPMNameParam \
		   | sddsconvert -pipe -del=col,OldColumnNames -del=para,* \
		   | sddscombine -pipe=in $outputFile -merge} result] {
	#------ Try again:
	if [catch {exec sddschanges $measuredFile -pipe=out -base=$computedFile -copy=BPMName -changesIn=exclude=BPMName,* \
		       | sddscollect -pipe -nowarning -collect=prefix=ChangeIn \
		       | sddsxref -pipe $tmpRoot.collect -take=Orbit -match=Rootname \
		       | sddsconvert -pipe -rename=col,ChangeIn=Error \
		       | tee $tmpRoot.errors \
		       | sddspfit -pipe -col=Orbit,Error -orders=0,1,3 -generate \
		       | sddsxref -pipe $tmpRoot.errors -transfer=para,BPMName -rename=para,BPMName=BPMNameParam \
		       | sddsconvert -pipe -del=col,* \
		       | sddsarray2column -pipe -convert=Coefficient \
		       | sddsxref -pipe $tmpRoot.col -take=CoefficientName -reuse=page \
		       | sddstranspose -pipe -newColumnName=CoefficientName \
		       | sddsprocess -pipe -print=col,BPMName,%s,BPMNameParam \
		       | sddsconvert -pipe -del=col,OldColumnNames -del=para,* \
		       | sddscombine -pipe=in $outputFile -merge} result] {
	    return -code error "Error performing nonlinear fitting - some BPMs might have zero readings only \
                            (measuredFile: $measuredFile; computedFile: $computedFile; tmpRoot: $tmpRoot): $result"
	}
    }
#    file delete $tmpRoot.collect $tmpRoot.col $tmpRoot.errors $measuredFile $computedFile
}

#-------------------------------------------------------------------------------------------------------------------------

proc SubtractMatricesAndMakeOneColumn { args } {
    
    APSParseArguments {hYes vYes hrmMeasured vrmMeasured hrmIterFile vrmIterFile hrmDifference vrmDifference \
                           xyColumnFile useQuadConstraints quadConstraintWeight coupledMatrix}
    
    global tmpDir dispColumn weightFile offDiagWeight options
    
    set tmpRoot $tmpDir/[APSTmpString]-make1col
    set planeFlagList [list $hYes $vYes]
    set measuredFileList [list $hrmMeasured $vrmMeasured]
    set iterFileList [list $hrmIterFile $vrmIterFile]
    set planeList [list X Y]
    set differenceFileList [list $hrmDifference $vrmDifference]
    set deleteFiles ""
    foreach planeFlag $planeFlagList measuredFile $measuredFileList iterFile $iterFileList plane $planeList \
        differenceFile $differenceFileList {
            if $planeFlag {
                if [catch {exec sddschanges $measuredFile -pipe=out -base=$iterFile -copy=BPMName -changes=exclude=BPMName,* \
			       -parallelPages \
			       | sddsconvert -pipe=in $differenceFile "-editNames=col,Change*,a 8d"} result] {
		    return -code error "Error calculating difference (measuredFile - $measuredFile; iterFile - $iterFile): $result"
		}
                if [catch {ApplyWeightCorrection -differenceFile $differenceFile -weightFile $weightFile \
                               -plane $plane} result] {
                    return -code error "ApplyWeightCorrection: $result"
                }
                if {[exec sdds2stream $differenceFile -npages=bare] == 2} {
                    #------ Applying offDiagWeight...
                    if {$plane == "X"} { set pageNum 2 } else { set pageNum 1 }
                    exec sddsprocess $differenceFile $tmpRoot.dif \
                        "-redef=col,%s,i_page $pageNum == pop pop ? %s $offDiagWeight * : %s \$,select=*,exclude=BPMName"
                } else {
                    file copy -force $differenceFile $tmpRoot.dif
                }
		lappend deleteFiles $tmpRoot.dif
                if [catch {Fit_MakeColumnFromMatrix -matrixFile $tmpRoot.dif -columnFile $tmpRoot.$plane \
                               -columnName MatrixDifference} result] {
                    return -code error "Fit_MakeColumnFromMatrix: $result"
                }
            } else {
                exec sddsmakedataset $tmpRoot.$plane -col=Rootname,type=string -data \
                    -col=MatrixDifference,type=double -data
            }
        }
    set combineFiles "$tmpRoot.X $tmpRoot.Y"

    #------ Vector of tune differences:
    if $options(useTune) {
	if [catch {MakeTunesColumn -filename $tmpRoot.tunes} result] {
	    return -code error "MakeTunesColumn: $result"
	}
	lappend combineFiles $tmpRoot.tunes
    }

    #------ Calculating vector of quad constraint additions...
    if $useQuadConstraints {
	global variableFile allElementsFile dontUsePrevQuadConstr
	set varFile $tmpRoot.QuadSkew
	lappend deleteFiles $tmpRoot.QuadSkew
	#------ allElementsFile contains data from previous run and might contain more elements
	if $dontUsePrevQuadConstr {set quadValueFile $variableFile} else {set quadValueFile $allElementsFile}
	if ![exec sdds2stream $quadValueFile.Quad -rows=bare] {
	    return -code error "Quad constraints should not be used if there are no quads to fit."
	}
	if {$coupledMatrix == 2 && $options(useSkew) && $useQuadConstraints == 2} {
	    exec sddsselect $quadValueFile.Quad $variableFile.Quad $tmpRoot.Quad -match=ElementName
	    exec sddsselect $quadValueFile.Skew $variableFile.Skew $tmpRoot.Skew -match=ElementName
	    exec sddscombine $tmpRoot.Quad $tmpRoot.Skew $varFile -merge -overWrite
	    lappend deleteFiles $tmpRoot.Quad $tmpRoot.Skew
	} else {
	    exec sddsselect $quadValueFile.Quad $variableFile.Quad $varFile -match=ElementName
	}
	if [catch {exec sddsconvert $varFile -pipe=out \
		       -retain=col,ElementName,ParameterValue -rename=col,ElementName=Rootname,ParameterValue=MatrixDifference \
		       | sddsprocess -pipe=in $tmpRoot.quads \
		       "-redef=col,MatrixDifference,MatrixDifference $quadConstraintWeight * -1 *"} $result] {
	    return -code error "Error making constraint vector (varFile - $varFile): $result"
	}
	lappend combineFiles $tmpRoot.quads
	set constraintLines [exec sdds2stream $tmpRoot.quads -rows=bare]
    } else {
	set constraintLines 0
    }
    eval exec sddscombine $combineFiles -pipe=out -merge \
	| sddsprocess -pipe=in $xyColumnFile -redef=para,ConstraintLines,$constraintLines,type=long

    if [catch {CalculateResidualRms -vectorX $tmpRoot.X -vectorY $tmpRoot.Y \
                   -measuredFileX $hrmMeasured -measuredFileY $vrmMeasured -coupledMatrix $coupledMatrix \
		   -vectorQuads $tmpRoot.quads -quadConstraintWeight $quadConstraintWeight} rmsList] {
        return -code error "CalculateResidualRms: $rmsList"
    }
    eval file delete $combineFiles $deleteFiles
    return $rmsList
}

#-------------------------------------------------------------------------------------------------------------------------

proc ApplyWeightCorrection {args} {

    APSParseArguments {differenceFile weightFile plane}

    global tmpDir useBpmWeight weightCorrectorsFile sigmaWeightFileX sigmaWeightFileY useSigmaWeightFile
    
    set deleteFiles ""
    if $useBpmWeight {
        set tmpFile $tmpDir/[APSTmpString].weight
        if {[exec sdds2stream $differenceFile -npages=bare] != 2} {
            exec sddsprocess $weightFile $tmpFile -match=para,Plane=$plane
        } else {
            file copy -force $weightFile $tmpFile
        }
        exec sddsxref $tmpFile $differenceFile -pipe=out -leave=BPMName -match=BPMName -nowarning \
            | sddsconvert -pipe -del=col,BPMName \
            | sddsprocess -pipe "-redef=col,%s,%s Weight *,select=*,exclude=Weight" \
            | sddsconvert -pipe=in $tmpFile.1 -del=col,Weight
        exec sddsxref $tmpFile.1 $differenceFile $tmpFile.2 -take=BPMName
        file copy -force $tmpFile.2 $differenceFile
        lappend deleteFiles $tmpFile $tmpFile.1 $tmpFile.2
    }
    
    #------ Adjusting for corrector weights...
    if [file exists $weightCorrectorsFile] {
        OutputStatusMessage "Applying corrector weight..."
        set correctorList [join [exec sdds2stream $weightCorrectorsFile -col=Corrector] ]
        set weightList [join [exec sdds2stream $weightCorrectorsFile -col=Weight] ]
        set columnList [join [exec sddsquery -col $differenceFile] ]
        set redefLine ""
        foreach corrector $correctorList weight $weightList {
            if {[lsearch -exact $columnList $corrector] != -1} {
                append redefLine " \"-redef=col,$corrector,$corrector $weight *\" "
            }
        }
        if [string length $redefLine] {
            eval exec sddsprocess -nowarning $differenceFile $redefLine
            lappend deleteFiles ${differenceFile}~
        }
    }

    #------ Applying sigmaWeightFile:
    if $useSigmaWeightFile {
	switch -exact $plane {
	    X {set sigmaWeightFile $sigmaWeightFileX}
	    Y {set sigmaWeightFile $sigmaWeightFileY}
	    default {return -code error "Wrong plane variable: $plane"}
	}
	if [file exists $sigmaWeightFile] {
	    set tmpFile $tmpDir/[APSTmpString].sigmaweight
	    #------ sddsmatrixop deletes all string columns.
#???	    file copy -force $differenceFile $tmpFile.0
	    exec sddsmatrixop $differenceFile -pipe=out -push=$sigmaWeightFile -divide=hadamard \
		| sddsxref -pipe=in $differenceFile $tmpFile -take=BPMName
	    file copy -force $tmpFile $differenceFile
	    lappend deleteFiles $tmpFile
	}
    }
    catch {eval file delete $deleteFiles}
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateResidualRms { args } {
    
    global dispColumn thetaKickX thetaKickY dispFitWeight offDiagWeight options
    APSParseArguments {vectorX vectorY measuredFileX measuredFileY vectorQuads quadConstraintWeight coupledMatrix}
    
    set vectorFileList [list $vectorX $vectorY]
    set thetaKickList [list $thetaKickX $thetaKickY]
    set measuredFileList [list $measuredFileX $measuredFileY]
    set planeList [list X Y]
    foreach vectorFile $vectorFileList thetaKick $thetaKickList measuredFile $measuredFileList plane $planeList {
        if [exec sddscombine $vectorFile -pipe=out -merge | sdds2stream -pipe=in -rows=bare] {
            #------ Undoing offDiagWeight before calculating rms...
            if {[exec sdds2stream $vectorFile -npages=bare] == 2} {
                if {$plane == "X"} { set pageNum 2 } else { set pageNum 1 }
                exec sddsprocess $vectorFile $vectorFile.tmp \
                    "-redef=col,%s,i_page $pageNum == pop pop ? %s $offDiagWeight / : %s \$,select=*,exclude=Rootname"
            } else {
                file copy -force $vectorFile $vectorFile.tmp
            }
            #------ This is for dispersion-dependent processing...
            if {[lsearch -exact [exec sddsquery -col $measuredFile] $dispColumn] != -1 } {
                #------ Orbit rms in mm (transformation from m/rad to mm).
                set stdDevZ [exec sddsprocess $vectorFile.tmp -pipe=out "-match=col,Rootname=${dispColumn}*,!" \
                                 | sddsprocess -pipe \
                                 "-redef=col,MatrixDifference,MatrixDifference $thetaKick 1000 * *" \
                                 | tee $vectorFile.Zmm \
                                 | sddsprocess -pipe -process=MatrixDifference,standardDeviation,stdDev \
                                 | sdds2stream -pipe=in -para=stdDev]
                #------ Dispersion rms in m.
                set stdDevD [exec sddsprocess $vectorFile.tmp -pipe=out "-match=col,Rootname=${dispColumn}*" \
                                 | sddsprocess -pipe \
                                 "-redef=col,MatrixDifference,MatrixDifference $dispFitWeight /" \
                                 | tee $vectorFile.Dm \
                                 | sddsprocess -pipe -process=MatrixDifference,standardDeviation,stdDev \
                                 | sdds2stream -pipe=in -para=stdDev]
                set stdDev$plane [list $stdDevZ $stdDevD]
                exec sddscombine $vectorFile.Zmm $vectorFile.Dm $vectorFile.mm -merge
                file delete $vectorFile.Zmm $vectorFile.Dm
            } else {
                exec sddsprocess $vectorFile.tmp $vectorFile.mm \
                    "-redef=col,MatrixDifference,MatrixDifference $thetaKick 1000 * *"
                if [catch {exec sddsprocess $vectorFile.mm -pipe=out \
                               -process=MatrixDifference,standardDeviation,stdDev \
                               | sdds2stream -pipe=in -para=stdDev} stdDevZ] {
                    return -code error "Error processing file $vectorFile.mm: $stdDevZ"
                }
                set stdDev$plane [list $stdDevZ]
            }
            file delete $vectorFile.tmp
        } else {
            set stdDev$plane Nan
            file copy -force $vectorFile $vectorFile.mm
        }
    }
    set combineFiles "$vectorX.mm $vectorY.mm"

    #------ Calculating quad constraint std...
    if [file exists $vectorQuads] {
	#------ Find stdev without quads first:
	if [catch {eval exec sddscombine $combineFiles -pipe=out -merge \
		       | sddsprocess -pipe -process=MatrixDifference,standardDeviation,stdDev \
		       | sdds2stream -pipe=in -para=stdDev} stdDev1] {
	    return -code error "Error combining files (1) ($combineFiles): $stdDev1"
	}
	#------ Divide by weight (actually need to involve quadConstraintFile here):
	if [catch {exec sddsprocess $vectorQuads -pipe=out \
		       "-redef=col,MatrixDifference,MatrixDifference $quadConstraintWeight /" \
		       -process=MatrixDifference,standardDeviation,StdDev \
		       | tee $vectorQuads.mm \
		       | sdds2stream -pipe=in -para=StdDev} stdDevQ] {
	    return -code error "Error calculating quad vector: $stdDevQ"
	}
	lappend combineFiles $vectorQuads.mm
    }
    
    if [catch {eval exec sddscombine $combineFiles -pipe=out -merge \
		   | sddsprocess -pipe -process=MatrixDifference,standardDeviation,stdDev \
		   | sdds2stream -pipe=in -para=stdDev} stdDev] {
	return -code error "Error combining files (2) ($combineFiles): $stdDev"
    }
    eval file delete $combineFiles
    
    set returnList ""
    if {$coupledMatrix == 2} {
	set returnList [concat $returnList "rmsYX [lindex [lindex $stdDevY 0] 0] rmsYY [lindex [lindex $stdDevY 0] 1]"]
	set returnList [concat $returnList "rmsXX [lindex [lindex $stdDevX 0] 0] rmsXY [lindex [lindex $stdDevX 0] 1]"]
	if $options(fitDispersion) {
	    set returnList [concat $returnList "rmsDX [lindex [lindex $stdDevX 1] 0] rmsDY [lindex [lindex $stdDevX 1] 1]"]
	}
    } else {
	set returnList [concat $returnList "rmsYY $stdDevY"]
	if $options(fitDispersion) {
	    set returnList [concat $returnList "rmsXX [lindex $stdDevX 0] rmsDX [lindex $stdDevX 1]"]
	} else {
	    set returnList [concat $returnList "rmsXX [lindex $stdDevX 0]"]
	}
    }
    if [file exists $vectorQuads] {
	set returnList [concat $returnList "rmsTotal $stdDev rmsTotalWQ $stdDev1 rmsQ $stdDevQ"]
    } else {
	set returnList [concat $returnList "rmsTotal $stdDev"]
    }
    return $returnList
}

#-------------------------------------------------------------------------------------------------------------------------

proc MakeTunesColumn {args} {
    APSParseArguments {filename}
    global tmpDir nuxMeasured nuyMeasured twiFile
    set calculatedTunes [exec sdds2stream -para=nux,nuy $twiFile]
    if {[lsearch -glob $calculatedTunes *NaN*] != -1} {
        return -code error "Error: Tunes are unstable!"
    }
    exec sddsmakedataset -pipe=out \
	-col=Rootname,type=string -data=Snux,Snuy \
	-col=CalcTunes,type=double -data=[join $calculatedTunes ,] \
	-col=MeasTunes,type=double -data=$nuxMeasured,$nuyMeasured \
	| sddsprocess -pipe "-redef=col,MatrixDifference,MeasTunes CalcTunes -" \
	| sddsconvert -pipe=in $filename -del=col,CalcTunes,MeasTunes
    OutputStatusMessage [format "%s %10.4f %10.4f" "Calculated Tunes are " [lindex $calculatedTunes 0] [lindex $calculatedTunes 1]]
}

#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
#------ Response Matrix Derivative related Procedures and Matlab related Procedures -----------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------

proc BuildResponseMatrixDeriv { args } {
    
    APSParseArguments { continuePrevious abortFile rmDerivFile useQsubLong hrmFile vrmFile lteFile optimFile \
                            coupledMatrix acceleratorCode hrmXFile vrmXFile beamlineName solutionParamFileList \
			    useKickFiles kickFileX kickFileY}
    
    global elementListVar elementListRM env rmDerivElementFile
    global latticeIterFile latticeRMFile dispersionFile twiFile variableFile rmDifferenceFile
    global options kickRMD fixedOrbitLength directRMCalc dispColumn dispFitWeight optimNonlin bRho 
    global waitTimeLong waitIntervalLong numberOfQsubTasksLong useDoubleOrbits
    global workDir tmpDir locoBinDir locoSearchString qsubCommandLong submissionPauseLong
    global hYes vYes makeSkewNamesUnique closedOrbit qsubRespProcCommandLong queueSystemNameLong
    global rmdQuadDelta rmdTiltDelta remotePath verbose
    global twissRefElement twissRefFile malignElement specialElementsFile
    
    if ![file exists $workDir/dqs] {exec mkdir $workDir/dqs}
    
    OutputStatusMessage "Doing the orbit response matrix derivative..."

    set uniqueMatrixCode [APSTmpString]
    if [catch {MakeRMDerivElementListFile -varList $elementListVar -rmList $elementListRM -outputFile $rmDerivElementFile \
		   -uniqueMatrixCode $uniqueMatrixCode -coupledMatrix $coupledMatrix -rmDerivFile $rmDerivFile} result] {
	return -code error "MakeElementListFile: $result"
    }

    set deleteFiles ""
    set matrixDoneList ""
    set matrixErrorList ""
    set matrixArrayList ""
    set hrmFile00 $workDir/computed.hrm.000
    set vrmFile00 $workDir/computed.vrm.000
    exec sddscombine $hrmFile $hrmFile00 -merge -overWrite
    exec sddscombine $vrmFile $vrmFile00 -merge -overWrite
    lappend deleteFiles $hrmFile00 $vrmFile00
    set tmpRoot $tmpDir/[APSTmpString]-RMD
    set matrixAbortFile $tmpRoot.abort
    switch -exact -- $coupledMatrix {
        0 { 
	    set indexList [list useQuads useCorrs useBPMs useEnergy] 
	    set arrayIndexList [list quads hCorr vCorr xBpm yBpm energy]
	}
        1 { 
	    set indexList [list useSkew useCorrsTilt useBPMsTilt] 
	    set arrayIndexList [list skew hCorrTilt vCorrTilt xBpmTilt yBpmTilt]
	}
        2 { 
	    set indexList [list useQuads useSkew useCorrs useBPMs useEnergy useCorrsTilt useBPMsTilt] 
	    set arrayIndexList [list quads hCorr vCorr xBpm yBpm energy skew hCorrTilt vCorrTilt xBpmTilt yBpmTilt]
	}
    }
    if ![file exists $workDir/dqs] {exec mkdir $workDir/dqs}
    
    foreach index $indexList {
        if $options($index) {
            switch -regexp -- $index {
                useQuads|useSkew {
                    switch -exact -- $index {
                        useQuads {
                            OutputStatusMessage "Doing quadrupoles..."
			    set ext quads
                            set varListIndex 0
                            set elementParameter K1
			    set parameterDelta $rmdQuadDelta
			    set colPrefix ""
			    set colSuffix ""
                        }
                        useSkew {
                            OutputStatusMessage "Doing skew quadrupoles..."
			    set ext skew
                            set varListIndex 6
                            set elementParameter TILT
			    set parameterDelta $rmdTiltDelta
			    if $makeSkewNamesUnique { set colPrefix a }
			    set colSuffix ""
                        }
                        default { return -code error "Error in index: index= $index" }
                    }
		    if [string length $remotePath] {set remoteQueue 1} else {set remoteQueue 0}
		    set tmpDirName $workDir/dqs/$ext
		    set localTmpDir $tmpDir/$ext
		    set remotePath1 $remotePath/$ext
		    set matrixFile $tmpRoot.$ext
		    set rootTaskName $ext
		    #------ Need "dummy" here because in string scriptParameters zero length variable means just empty space.
		    set possibleEmptyVars [list lteFile beamlineName solutionParamFileList optimFile twissRefFile twissRefElement \
					       malignElement kickFileX kickFileY]
		    foreach var $possibleEmptyVars {if ![string length [set $var]] {set $var dummy}}

                    set scriptName $locoBinDir/runQuads
                    set scriptParameters    "-parameterDelta $parameterDelta -acceleratorCode $acceleratorCode "
                    append scriptParameters "-tuneInclude $options(useTune) -kick $kickRMD -bRho $bRho "
                    append scriptParameters "-fixedOrbitLength $fixedOrbitLength -fitDispersion $options(fitDispersion) "
                    append scriptParameters "-dispColumn $dispColumn -dispFitWeight $dispFitWeight "
                    append scriptParameters "-coupledMatrix $coupledMatrix -locoSearchString $locoSearchString "
                    append scriptParameters "-directRMCalc $directRMCalc -optimNonlin $optimNonlin "
                    append scriptParameters "-makeBaselineCalc 1 -elementParameter $elementParameter "
		    append scriptParameters "-lteFile $lteFile -beamlineName $beamlineName -optimFile $optimFile "
		    append scriptParameters "-computedRMX $hrmFile00 -computedRMY $vrmFile00 -closedOrbit $closedOrbit "
		    append scriptParameters "-useDoubleOrbits $useDoubleOrbits -latticeParamFileList \"$solutionParamFileList\" "
		    append scriptParameters "-twissRefFile $twissRefFile -twissRefElement $twissRefElement \
                                             -malignElement $malignElement  "
		    append scriptParameters "-useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY "
		    append scriptParameters "-localTmpDir $localTmpDir "
                    set varList [lindex $elementListVar $varListIndex]
                    if ![llength $varList] {
                        OutputStatusMessage \
                            "BuildResponseMatrixDeriv Warning: No variables specified for $index option \
                             (elementListVar: $elementListVar)"
                    } else {
			set doneFile $matrixFile.done
			set errorFile $matrixFile.error
			set optionsFile $matrixFile.options
			if [catch {CalculateOrbitRMD -varList $varList \
				       -splitTasks $numberOfQsubTasksLong -colPrefix $colPrefix -colSuffix $colSuffix \
				       -scriptName $scriptName -scriptParameters $scriptParameters \
				       -matrixFile $matrixFile -tmpDirName $tmpDirName -rootTaskName $rootTaskName \
				       -useQsub $useQsubLong -continuePrevious $continuePrevious -usePopupWindow 0 \
				       -waitTime $waitTimeLong -qsubCommand $qsubCommandLong \
				       -waitInterval $waitIntervalLong -submissionPause $submissionPauseLong \
				       -waitForCompletion 0 -doneFile $doneFile -errorFile $errorFile \
				       -matrixAbortFile $matrixAbortFile -optionsFile $optionsFile -remotePath $remotePath1 \
				       -lteFile $lteFile -beamlineName $beamlineName -optimFile $optimFile \
				       -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY \
				       -solutionParamFileList $solutionParamFileList -remoteQueue $remoteQueue \
				       -qsubRespProcCommand $qsubRespProcCommandLong -queueSystemName $queueSystemNameLong \
				       -abortFile $abortFile} result] {
			    return -code error "CalculateOrbitRMD: $result"
			}
		    }
		    lappend matrixDoneList $doneFile; lappend deleteFiles $doneFile
		    lappend matrixErrorList $errorFile
		    lappend matrixArrayList $ext $matrixFile
		}
                useCorrs|useCorrsTilt {
                    set planeList [list X Y]
		    set mode cor
		    set parameterDelta 1e-4
                    switch -exact -- $index {
                        useCorrs {
			    OutputStatusMessage "Doing correctors..."
                            set extList [list hCorr vCorr]
			    set colPrefix ""
			    set colSuffix ""
			    if $useKickFiles {
				set elementParameter CALIBRATION
				set tmpExt corrs
			    } else {
				set rmFileList [list $hrmFile00 $vrmFile00]
			    }
			}
                        useCorrsTilt {
                            OutputStatusMessage "Doing corrector tilts..."
                            set extList [list hCorrTilt vCorrTilt]
			    set colPrefix a
			    set colSuffix ""
			    if $useKickFiles {
				set elementParameter TILT
				set tmpExt corrTilts
			    } else {
				set corrTiltXRMDSource $tmpRoot.corrTiltX
				set corrTiltYRMDSource $tmpRoot.corrTiltY
				set rmFileList [list $corrTiltXRMDSource $corrTiltYRMDSource]
				lappend deleteFiles $corrTiltXRMDSource $corrTiltYRMDSource
				#------ Preparing files for RMD:
				if [catch {CalculateTiltedCorrResponse -lteFile $lteFile -beamlineName $beamlineName \
					       -latticeParamFileList $solutionParamFileList -optimFile $optimFile \
					       -coupledMatrix $coupledMatrix -corrTiltXRMDSource $corrTiltXRMDSource \
					       -corrTiltYRMDSource $corrTiltYRMDSource} result] {
				    return -code error "CalculateTiltedCorrResponse: $result"
				}
			    }
                        }
		    }
		    if $useKickFiles {
			set tmpDirName $workDir/dqs/$tmpExt
			set localTmpDir $tmpDir/$tmpExt
			if ![file exists $tmpDirName] {exec mkdir $tmpDirName}
			set corrListX [Fit_DeleteElementsFromList -elementList [lindex $elementListVar 1]  \
					   -deleteElements $dispColumn]
			set corrListY [lindex $elementListVar 2]
			set pathList [list $workDir/dqs/$tmpExt/X $workDir/dqs/$tmpExt/Y]
			foreach varList [list $corrListX $corrListY] path $pathList ext $extList {
			    if ![file exists $path] {exec mkdir $path}
			    set matrixFile $tmpRoot.$ext
			    set command "$locoBinDir/runQuads -argMode direct -verbose $verbose \
					   -path $path -varList \"$varList\" \
					   -parameterDelta $parameterDelta -acceleratorCode $acceleratorCode \
					   -tuneInclude $options(useTune) -kick $kickRMD -bRho $bRho \
					   -fixedOrbitLength $fixedOrbitLength -fitDispersion $options(fitDispersion) \
					   -dispColumn $dispColumn -dispFitWeight $dispFitWeight \
					   -coupledMatrix $coupledMatrix -locoSearchString $locoSearchString \
					   -directRMCalc $directRMCalc -optimNonlin $optimNonlin \
					   -makeBaselineCalc 1 -elementParameter $elementParameter \
					   -lteFile $lteFile -beamlineName $beamlineName -optimFile $optimFile \
					   -computedRMX $hrmFile00 -computedRMY $vrmFile00 -closedOrbit $closedOrbit \
					   -useDoubleOrbits $useDoubleOrbits \
					   -latticeParamFileList \"$solutionParamFileList\" \
					   -twissRefFile $twissRefFile -twissRefElement $twissRefElement \
					   -malignElement $malignElement  \
					   -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY \
					   -localTmpDir $localTmpDir"
			    OutputStatusMessage $command -logFileOnly 1
			    if [catch {eval exec $command >@ stdout} result] {
				return -code error "$locoBinDir/runQuads: $result"
			    }
			    file copy -force $path/responseMatrix.sdds $matrixFile
			    lappend matrixArrayList $ext $matrixFile
			    if [catch {ChangeColumnNames -matrixFile $matrixFile \
					   -colPrefix $colPrefix -colSuffix $colSuffix} result] {
				return -code error "ChangeColumnNames: $result"
			    }
			}
			if {$options(fitDispersion) && [string compare useCorrs $index] == 0} {
			    exec sddsconvert $hrmFile00 -pipe=out -del=col,BPMName,s \
				| sddsprocess -pipe "-redef=col,%s,0,select=*,exclude=$dispColumn,units=m/rad" \
				| sddsxref -pipe $hrmFile00 -take=BPMName \
				| sddscombine -pipe=in $tmpRoot.dispDiff -merge
			    if [catch {Fit_MakeColumnFromMatrix -matrixFile $tmpRoot.dispDiff \
					   -columnFile $tmpRoot.dispColumn -columnName $dispColumn} result] {
				return -code error "Fit_MakeColumnFromMatrix: %result"
			    }
			    exec sddsxref $tmpRoot.hCorr $tmpRoot.dispColumn $tmpRoot.hCorr1 -take=$dispColumn \
				-nowarning -fillIn
			    file copy -force $tmpRoot.hCorr1 $tmpRoot.hCorr
			    lappend deleteFiles $tmpRoot.dispDiff $tmpRoot.dispColumn $tmpRoot.hCorr1
			}
			continue
		    } else {
			foreach planeFlag [list $hYes $vYes] plane $planeList rmFile $rmFileList ext $extList {
			    set matrixFile $tmpRoot.$ext
			    if $planeFlag {
				set outputFile $tmpRoot.$ext
				if [catch {QuickRMDeriv -mode $mode -plane $plane -rmFile $rmFile \
					       -variableFile $variableFile.$ext -outputFile $matrixFile \
					       -coupledMatrix $coupledMatrix} result] {
				    return -code error "QuickRMDeriv: $result"
				}
				OutputStatusMessage "QuickRMDeriv is done." -logFileOnly 1
				lappend matrixArrayList $ext $matrixFile
				if [catch {ChangeColumnNames -matrixFile $matrixFile \
					       -colPrefix $colPrefix -colSuffix $colSuffix} result] {
				    return -code error "ChangeColumnNames: $result"
				}
			    }
			}
		    }
		}
                useBPMs|useBPMsTilt {
                    set planeList [list X Y]
                    switch -exact -- $index {
                        useBPMs {
                            set mode bpm
                            set extList [list xBpm yBpm]
                            set rmFileList [list $hrmFile00 $vrmFile00]
			    set colPrefix ""
			    set colSuffix ""
                            OutputStatusMessage "Doing bpms..."
                        }
                        useBPMsTilt {
                            set mode bpmTilt
                            set extList [list xBpmTilt yBpmTilt]
			    set bpmTiltRMDSource1 $tmpRoot.bpmTilt1
			    set bpmTiltRMDSource2 $tmpRoot.bpmTilt2
                            set rmFileList [list $bpmTiltRMDSource2 $bpmTiltRMDSource1]
			    lappend deleteFiles $bpmTiltRMDSource1 $bpmTiltRMDSource2
			    set colPrefix a
			    set colSuffix ""
                            OutputStatusMessage "Doing bpm tilts..."
			    #------ Preparing files for calculation of RM derivative with respect to BPM tilts:
			    #------ Assuming Xtilted =  X*cos(delta)+Y*sin(delta) =  X+Y*delta
			    #------          Ytilted = -X*sin(delta)+Y*cos(delta) = -X*delta+Y
			    #------ the files are cross rm files with signes of the above (+ +; - +).
			    #------ Signes are checked and confirmed.
			    exec sddsprocess $hrmXFile $bpmTiltRMDSource1 \
				"-redef=col,%s,i_page 1 == ? %s : %s -1 * \$,select=*,exclude=BPMName"
			    exec sddsprocess $vrmXFile $bpmTiltRMDSource2 \
				"-redef=col,%s,i_page 1 == ? %s : %s -1 * \$,select=*,exclude=BPMName"
                        }
                    }
                    foreach planeFlag [list $hYes $vYes] plane $planeList rmFile $rmFileList ext $extList {
                        if $planeFlag {
                            set matrixFile $tmpRoot.$ext
                            if [catch {QuickRMDeriv -mode $mode -plane $plane -rmFile $rmFile \
                                           -variableFile $variableFile.$ext -outputFile $matrixFile \
                                           -coupledMatrix $coupledMatrix} result] {
                                return -code error "QuickRMDeriv: $result"
                            }
                            OutputStatusMessage "QuickRMDeriv is done." -logFileOnly 1
			    lappend matrixArrayList $ext $matrixFile
			    if [catch {ChangeColumnNames -matrixFile $matrixFile \
					   -colPrefix $colPrefix -colSuffix $colSuffix} result] {
				return -code error "ChangeColumnNames: $result"
			    }
                        }
                    }
                }
		useEnergy {
                    if $hYes {
                        OutputStatusMessage "Doing energy..."
                        set rmFile $tmpDir/dispersion.Xmatrix
                        set matrixFile $tmpRoot.energy
			set ext energy
			set colPrefix ""
			set colSuffix E
                        #------ Create file with dimensions of hrmFile and with all columns equal to dispColumn from dispFile.
                        exec sddscombine $dispersionFile $tmpRoot.dispersion -merge -overWrite
                        exec sddsxref $hrmFile00 $tmpRoot.dispersion -pipe=out -take=$dispColumn \
                            -match=BPMName=ElementName -rename=col,$dispColumn=${dispColumn}1 -nowarning \
                            | sddsprocess -pipe "-redef=col,%s,${dispColumn}1,select=*,exclude=BPMName" \
                            | sddsconvert -pipe=in $rmFile -del=col,${dispColumn}1
                        if [catch {QuickRMDeriv -mode cor -plane X -rmFile $rmFile \
                                       -variableFile $variableFile.Energy \
                                       -outputFile $matrixFile \
                                       -coupledMatrix $coupledMatrix} result] {
                            return -code error "QuickRMDeriv: $result"
                        }
			lappend matrixArrayList $ext $matrixFile
			if [catch {ChangeColumnNames -matrixFile $matrixFile \
				       -colPrefix $colPrefix -colSuffix $colSuffix} result] {
			    return -code error "ChangeColumnNames: $result"
			}
                        lappend deleteFiles $rmFile $tmpRoot.dispersion
                    }
                }
                default {
                    OutputStatusMessage "BuildResponseMatrixDeriv: Warning: wrong option $index."
                }
            }
        }
    }
    if [file exists $specialElementsFile] {
	if [exec sdds2stream $specialElementsFile -rows=bare] {
	    OutputStatusMessage "Doing special elements..."
	    set matrixFile $tmpRoot.special
	    set specialElementMatrixList ""
	    set specialElementList [join [exec sdds2stream $specialElementsFile -col=ElementName] ]
	    set specialElementScripts [join [exec sdds2stream $specialElementsFile -col=ScriptName] ]
	    set specialElementOptions [join [exec sdds2stream $specialElementsFile -col=ScriptOptions] ]
	    set specialElementDeltas [join [exec sdds2stream $specialElementsFile -col=Delta] ]
	    set specialElementSuffixes [join [exec sdds2stream $specialElementsFile -col=NameSuffix] ]
	    
	    set localTmpDir $tmpDir/special
	    if ![file exists $localTmpDir] {exec mkdir $localTmpDir}
	    if ![file exists $workDir/dqs/special] {exec mkdir $workDir/dqs/special}
	    set mainScript $locoBinDir/runRMDerivCalc
	    foreach specialElement $specialElementList specialScript $specialElementScripts specialOptions $specialElementOptions \
		specialDelta $specialElementDeltas suffix $specialElementSuffixes {
		    set totalPath $workDir/dqs/special/$specialElement
		    if ![file exists $totalPath] {exec mkdir $totalPath}
		    set localMatrixFile $totalPath/responseMatrix.sdds
		    set scriptParameters    "-acceleratorCode $acceleratorCode "
		    append scriptParameters "-tuneInclude $options(useTune) -kick $kickRMD -bRho $bRho "
		    append scriptParameters "-fixedOrbitLength $fixedOrbitLength -fitDispersion $options(fitDispersion) "
		    append scriptParameters "-dispColumn $dispColumn -dispFitWeight $dispFitWeight "
		    append scriptParameters "-coupledMatrix $coupledMatrix -locoSearchString $locoSearchString "
		    append scriptParameters "-directRMCalc $directRMCalc -optimNonlin $optimNonlin "
		    append scriptParameters "-makeBaselineCalc 1 "
		    append scriptParameters "-lteFile $lteFile -beamlineName $beamlineName -optimFile $optimFile "
		    append scriptParameters "-optimRMFile $totalPath/latticeRM.file "
		    append scriptParameters "-computedRMX $hrmFile00 -computedRMY $vrmFile00 -closedOrbit $closedOrbit "
		    append scriptParameters "-useDoubleOrbits $useDoubleOrbits -latticeParamFileList \"$solutionParamFileList\" "
		    append scriptParameters "-twissRefFile $twissRefFile -twissRefElement $twissRefElement -malignElement $malignElement "
		    append scriptParameters "-useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY "
		    append scriptParameters "-localTmpDir $localTmpDir -parameterDelta $specialDelta -suffix $suffix "
		    append scriptParameters "-rmDerivCalcScript $specialScript -rmDerivCalcScriptOptions \"$specialOptions\" "
		
		    set scriptParametersFile $totalPath/scriptParameters
		    if [catch {open $scriptParametersFile w} fid] { 
			return -code error "$fid" 
		    }
		    puts $fid "-path $totalPath -varList $specialElement $scriptParameters"
		    close $fid
		    
		    puts stdout "$mainScript -argMode file -scriptParametersFile $scriptParametersFile -verbose $verbose"
		    if [catch {exec $mainScript -argMode file -scriptParametersFile $scriptParametersFile -verbose $verbose} result] {
			return -code error "$mainScript: $result"
		    }
		    lappend specialElementMatrixList $localMatrixFile
		}
	    if {[llength $specialElementMatrixList] == 1} {
		file copy -force $specialElementMatrixList $tmpRoot.special
	    } else {
		eval exec sddsxref $specialElementMatrixList $tmpRoot.special -take=* -leave=Rootname -nowarning
	    }
#	    eval file delete $specialElementMatrixList
	    lappend matrixArrayList special $matrixFile
	    lappend arrayIndexList special
	}
    }
    #------ Handling special variables:
    if $options(useAverGains) {
	OutputStatusMessage "Doing average gains..."
	if {$coupledMatrix != 2} {
	    return -code error "useAverGains option works for coupledMatrix=2 only."
	}
	set localTmpRoot $tmpRoot-averGain
	set matrixFile $tmpRoot.averGains
	foreach plane [list X Y] multListList [list [list "0 -1" "1 0"] [list "0 1" "-1 0"]] {
	    set columnFileList ""
	    foreach rmFile [list $hrmFile $vrmFile] multList $multListList ext [list 1 2] {
		set mult1 [lindex $multList 0]
		set mult2 [lindex $multList 1]
		exec sddsprocess $rmFile -pipe=out "-define=para,Multiplier,i_page 1 == ? $mult1 : $mult2 \$" \
		    "-redef=col,%s,%s Multiplier *,select=*,exclude=BPMName" \
		    | sddscombine -pipe=in $localTmpRoot.$plane.$ext -merge -overWrite
		if [catch {Fit_MakeColumnFromMatrix -matrixFile $localTmpRoot.$plane.$ext \
			       -columnFile $localTmpRoot.$plane.$ext.column -columnName AverGain$plane} result] {
		    return -code error "Fit_MakeColumnFromMatrix: $result"
		}
		lappend columnFileList $localTmpRoot.$plane.$ext.column
		lappend deleteFiles $localTmpRoot.$plane.$ext $localTmpRoot.$plane.$ext.column
	    }
	    eval exec sddscombine $columnFileList $localTmpRoot.$plane -merge -overWrite
	    lappend deleteFiles $localTmpRoot.$plane
	}
	#------ Combine two columns:
	exec sddsxref $localTmpRoot.X $localTmpRoot.Y $matrixFile -take=AverGainY
	if $options(useTune) {
	    exec sddsmakedataset $localTmpRoot.tunes \
		-col=Rootname,type=string -data=Snux,Snuy \
		-col=AverGainX,type=double -data=0,0 \
		-col=AverGainY,type=double -data=0,0
	    exec sddscombine $matrixFile $localTmpRoot.tunes $localTmpRoot.result -merge -overWrite
	    file copy -force $localTmpRoot.result $matrixFile
	    lappend deleteFiles $localTmpRoot.tunes $localTmpRoot.result
	}
	lappend matrixArrayList special $matrixFile
	lappend arrayIndexList special
    }

    #------ Waiting for the matrix done files to appear:
    if [llength $matrixDoneList] {
	OutputStatusMessage "Waiting for background jobs to continue..."
	set done 0
	set waitInterval [expr $waitIntervalLong * 1000]
	while {$done == 0} {
	    set done 1
	    update
	    after $waitInterval
	    #------ Checking if the done files exist...
	    foreach filename $matrixDoneList {
		if ![file exists $filename] { set done 0 }
	    }
	    #------ Checking if error files have appeared...
	    foreach filename $matrixErrorList {
		set existingFiles ""
		if [file exists $filename] { lappend existingFiles $filename }
		if [llength $existingFiles] {
		    return -code error "Error running background jobs: see files $existingFiles"
		}
	    }
	    #------ Aborting if requested...
	    if [file exists $abortFile] {
		Fit_WriteToFile -filename $matrixAbortFile -accessMode w -line "Error: Interrupted by user."
		return -code error "Interrupted by user."
	    }
	}
    }
    
    OutputStatusMessage "Final combining..."
    file delete $rmDerivFile
    if 1 {
	#------ Make properly sorted matrixFile list:
	array set matrixArray $matrixArrayList
	set matrixFileList ""
	set existingIndexList [array names matrixArray]
	foreach arrayIndex $arrayIndexList {
	    if {[lsearch -exact $existingIndexList $arrayIndex] != -1} {
		lappend matrixFileList $matrixArray($arrayIndex)
	    }
	}
	set constraintLines [exec sdds2stream $rmDifferenceFile -para=ConstraintLines]
	OutputStatusMessage "List of combined files: $matrixFileList" -logFileOnly 1
	eval exec sddsprocess $rmDifferenceFile -pipe=out -clip=0,$constraintLines \
	    | sddsconvert -pipe -retain=col,Rootname \
	    | sddsxref -pipe $matrixFileList -take=* -leave=Rootname \
	    | sddsprocess -pipe=in $rmDerivFile -print=para,MatrixCode,$uniqueMatrixCode \
	    -print=para,ElementFile,[file tail $rmDerivElementFile] -def=para,CoupledMatrixAnalysis,$coupledMatrix
    } else {
	set tmpDerivFile $tmpDir/[file tail $rmDerivFile]
	file delete $rmDerivFile $tmpDerivFile
	if {[llength $matrixArrayList] == 2} {
	    file copy -force $matrixFileList $tmpDerivFile
	} else {
	    array set matrixArray $matrixArrayList
	    set matrixFileList ""
	    set existingIndexList [array names matrixArray]
	    foreach arrayIndex $arrayIndexList {
		if {[lsearch -exact $existingIndexList $arrayIndex] != -1} {
		    lappend matrixFileList $matrixArray($arrayIndex)
		}
	    }
	    OutputStatusMessage "List of combined files: $matrixFileList" -logFileOnly 1
	    if [catch {eval exec nice -19 sddsxref $matrixFileList $tmpDerivFile \
			   "-take=* -leave=Rootname" -nowarning} result] {
		return -code error "Error during final combining: $result"
	    }
	}
	exec sddsprocess $tmpDerivFile $rmDerivFile -print=para,MatrixCode,$uniqueMatrixCode \
	    -print=para,ElementFile,[file tail $rmDerivElementFile] -def=para,CoupledMatrixAnalysis,$coupledMatrix
    }
    set deleteFiles [concat $deleteFiles $matrixFileList]
    eval file delete $deleteFiles
    
    OutputStatusMessage "The response matrix derivative is done."
    return 0
}

#-------------------------------------------------------------------------------------------------------------------------

proc MakeRMDerivElementListFile {args} {
    global tmpDir options variableTypeList rmRootList fullOptionsList
    APSParseArguments {outputFile varList rmList uniqueMatrixCode coupledMatrix rmDerivFile}
    set fullOptionsList [list useQuads useCorrs useCorrs useBPMs useBPMs useEnergy useSkew useCorrsTilt useCorrsTilt \
			     useBPMsTilt useBPMsTilt]
    set varRootList [list Quad HCorr VCorr HBPM VBPM Energy Skew HCorrTilt VCorrTilt HBPMTilt VBPMTilt]
    set rmRootList  [list HCorr VCorr HBPM VBPM]
    set tmpRoot $tmpDir/[APSTmpString]-rmderivElements
    set fileList ""
    foreach rootname $variableTypeList option $fullOptionsList elementList $varList varRoot $varRootList {
	if $options($option) {
	    exec sddsmakedataset $tmpRoot.VAR.$rootname -para=VarType,type=string -data=Var \
		-para=VarName,type=string -data=$rootname \
		-para=VarRootname,type=string -data=$varRoot \
		-col=ElementName,type=string -data=[join $elementList ,]
	    lappend fileList $tmpRoot.VAR.$rootname
	} else {
	    exec sddsmakedataset $tmpRoot.VAR.$rootname -para=VarType,type=string -data=Var \
		-para=VarName,type=string -data=$rootname \
		-para=VarRootname,type=string -data=$varRoot \
		-col=ElementName,type=string -data
	    lappend fileList $tmpRoot.VAR.$rootname
	}
    }
    foreach rootname $rmRootList elementList $rmList {
	exec sddsmakedataset $tmpRoot.RM.$rootname -para=VarType,type=string -data=RM \
	    -para=VarName,type=string -data=$rootname \
	    -para=VarRootname,type=string -data=$rootname \
	    -col=ElementName,type=string -data=[join $elementList ,]
	lappend fileList $tmpRoot.RM.$rootname
    }
    eval exec sddscombine $fileList -pipe=out \
	| sddsprocess -pipe=in $outputFile -print=para,MatrixCode,$uniqueMatrixCode -nowarning \
	"\"-print=para,options,[array get options]\"" -def=para,CoupledMatrixAnalysis,$coupledMatrix \
	-print=para,RMDFile,[file tail $rmDerivFile]
    eval file delete $fileList
}

#-------------------------------------------------------------------------------------------------------------------------

proc ChangeColumnNames {args} {
    APSParseArguments {matrixFile colPrefix colSuffix}
    exec sddsconvert $matrixFile -pipe=out "-editName=col,*,a i%$colPrefix% e i%$colSuffix%" \
	| sddsconvert -pipe=in $matrixFile.edit -rename=col,${colPrefix}Rootname${colSuffix}=Rootname
    file copy -force $matrixFile.edit $matrixFile
    file delete $matrixFile.edit
}

#-------------------------------------------------------------------------------------------------------------------------

proc RecombineResponseMatrixDeriv {args} {
    global tmpDir env locoBinDir
    APSParseArguments {input output constraintWeight constraintFile variableFile coupledMatrix useQuadConstraints}
    OutputStatusMessage "[exec date +%D--%T]: Recombining RMD with quad constraints using source file $input..."
    set tmpRoot $tmpDir/[APSTmpString]-recomb
    set deleteFiles ""
    set varNumberList [join [exec sdds2stream $variableFile.sdds -rows=bare] ]
    set quadNumber [lindex $varNumberList 0]
    set skewNumber [lindex $varNumberList 6]
    if {$coupledMatrix == 2 && $skewNumber && $useQuadConstraints == 2} {set doSkew 1} else {set doSkew 0}
    if $doSkew {
	exec sddsprocess $variableFile.Skew $tmpRoot.skew "-reedit=col,ElementName,i/a/"
	exec sddscombine $variableFile.Quad $tmpRoot.skew $tmpRoot.QuadSkew -merge -overWrite
	lappend deleteFiles $tmpRoot.skew
	set quadVariableFile $tmpRoot.QuadSkew
	lappend deleteFiles $tmpRoot.QuadSkew
	#------ Calculate number of columns before skew columns:
	set addColumnsBefore [expr [lindex $varNumberList 0] + [lindex $varNumberList 1] + [lindex $varNumberList 2] \
				  + [lindex $varNumberList 3] + [lindex $varNumberList 4] + [lindex $varNumberList 5]] 
    } else {
	set quadVariableFile $variableFile.Quad
    }

    set doConstrVector 1
    if [file exists $constraintFile] {
	exec sddsconvert $quadVariableFile -pipe=out -retain=col,ElementName \
	    | sddsxref -pipe $constraintFile -take=CW -match=ElementName -fillIn -nowarning \
	    | sddsprocess -pipe=in $tmpRoot.constrVector \
	    "-redef=col,CW,CW 0 == ? $constraintWeight : $constraintWeight CW * \$" 
	set doConstrVector 0
    }
    if $doConstrVector {
	exec sddsconvert $quadVariableFile -pipe=out -retain=col,ElementName \
	    | sddsprocess -pipe=in $tmpRoot.constrVector -def=col,CW,$constraintWeight
    }
    lappend deleteFiles $tmpRoot.constrVector
    if $doSkew {
	if [catch {exec $locoBinDir/rmDerivative.$env(HOST_ARCH) $input $tmpRoot.rmderiv1 -mode=quad \
		       -addRowsBefore=0 -addRowsAfter=$quadNumber} result] {
	    return -code error "Error calculating constraint rows (1): $result"
	}
	if [catch {exec $locoBinDir/rmDerivative.$env(HOST_ARCH) $input $tmpRoot.rmderiv2 -mode=quad \
		       -addRowsBefore=$addColumnsBefore -addRowsAfter=$skewNumber} result] {
	    return -code error "Error calculating constraint rows (2): $result"
	}
	if [catch {exec sddscombine $tmpRoot.rmderiv1 $tmpRoot.rmderiv2 -pipe=out -merge \
		       | sddsxref -pipe $tmpRoot.constrVector -take=CW,ElementName -rename=col,ElementName=Rootname1 \
		       | sddsprocess -pipe "-redef=col,%s,%s CW *,select=*,exclude=Rootname*" \
		       | sddsconvert -pipe -del=col,CW,Rootname \
		       | sddsconvert -pipe=in $tmpRoot.constraints -rename=col,Rootname1=Rootname} result] {
	    return -code error "Error calculating constraint rows: $result"
	}
	lappend deleteFiles $tmpRoot.rmderiv1 $tmpRoot.rmderiv2
    } else {
	if [catch {exec $locoBinDir/rmDerivative.$env(HOST_ARCH) $input -pipe=out -mode=quad \
		       -addRowsBefore=0 -addRowsAfter=$quadNumber \
		       | sddsxref -pipe $tmpRoot.constrVector -take=CW,ElementName -rename=col,ElementName=Rootname1 \
		       | sddsprocess -pipe "-redef=col,%s,%s CW *,select=*,exclude=Rootname*" \
		       | sddsconvert -pipe -del=col,CW,Rootname \
		       | sddsconvert -pipe=in $tmpRoot.constraints -rename=col,Rootname1=Rootname} result] {
	    return -code error "Error calculating constraint rows: $result"
	}
    }
    lappend deleteFiles $tmpRoot.constraints
    exec sddscombine $input $tmpRoot.constraints -pipe=out -merge \
	| sddsprocess -pipe=in $output -reprint=para,SourceFile,$input
    eval file delete $deleteFiles
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateOrbitRMD { args } {

    global locoBinDir workDir definitionFile closedOrbit env verbose
    set waitForCompletion 0

    APSParseArguments {varList splitTasks scriptName scriptParameters matrixFile tmpDirName useQsub qsubCommand \
				 continuePrevious rootTaskName usePopupWindow waitTime waitInterval colPrefix colSuffix \
				 submissionPause waitForCompletion remotePath doneFile matrixAbortFile optionsFile \
				 errorFile lteFile beamlineName solutionParamFileList remoteQueue optimFile \
				 abortFile qsubRespProcCommand queueSystemName useKickFiles kickFileX kickFileY}

    if {!$remoteQueue || !$useQsub} {
	#------ Running on local host...
	if ![file exists $tmpDirName] {exec mkdir $tmpDirName}
	OutputStatusMessage "Starting background job..."
	set command "exec $locoBinDir/remoteCalculateDerivative -mode localRun -workDir $workDir \
            -varList \"$varList\" -splitTasks $splitTasks -scriptName $scriptName \
             -scriptParameters \"[ProtectQuoteSigns $scriptParameters]\" \
            -matrixFile $matrixFile -tmpDirName $tmpDirName -rootTaskName $rootTaskName \
            -useQsub $useQsub -continuePrevious $continuePrevious -waitTime $waitTime -queueSystemName $queueSystemName \
            -qsubCommand \"[ProtectDollarSigns $qsubCommand]\" -qsubRespProcCommand \"[ProtectDollarSigns $qsubRespProcCommand]\" \
            -queueSystemName $queueSystemName -waitInterval $waitInterval -submissionPause $submissionPause \
            -colPrefix \"$colPrefix\" -colSuffix \"$colSuffix\" -doneFile $doneFile -errorFile $errorFile -verbose $verbose"
	OutputStatusMessage $command -logFileOnly 1
	eval $command &
	#------ If you submit 2 jobs (quad and skew), they might get errors without pause
	after 1000
    } else {
	#------ Running on remote host...
	set remoteWorkDir [string range $remotePath [expr [string first : $remotePath] + 1] [string length $remotePath]]
	set remoteHostname [string range $remotePath 0 [expr [string first : $remotePath] - 1]]
	set copyFiles ""

	set parameterFileList ""
	if [llength $solutionParamFileList] { 
	    foreach paramFile $solutionParamFileList {
		lappend parameterFileList $remoteWorkDir/[file tail $paramFile]
		lappend copyFiles $paramFile
	    }
	}

	#------ Reading variables in scriptOptions and change it:
	array set optionsArray $scriptParameters
	set optionsArray(-latticeParamFileList) $parameterFileList
	set changeNameList [list lteFile twissRefFile computedRMX computedRMY]
	if $useKickFiles {lappend changeNameList kickFileX kickFileY}
	foreach name $changeNameList {
	    #------ Order of next three lines is important
	    set $name $optionsArray(-$name)
	    lappend copyFiles [set $name]
	    set optionsArray(-$name) $remoteWorkDir/[file tail $optionsArray(-$name)]
	}
	switch -exact $remoteHostname {
	    weed {set optionsArray(-localTmpDir) /tmp/$env(USER)/[file tail $optionsArray(-localTmpDir)]}
	    default {return -code error "Remote host $remoteHostname is not supported."}
	}

	catch {file delete $doneFile $matrixAbortFile $optionsFile}
	
	set optionsList [list splitTasks useQsub continuePrevious rootTaskName usePopupWindow waitTime waitInterval submissionPause \
			    queueSystemName]
	if [catch {open $optionsFile w} fid] { 
	    return -code error "$fid" 
	}
	foreach option $optionsList {
	    puts $fid "set $option [set $option]"
	}
	puts $fid "set tmpDirName $remoteWorkDir"
	puts $fid "set scriptName $remoteWorkDir/[file tail $scriptName]"
	puts $fid "set matrixFile $remoteWorkDir/[file tail $matrixFile]"
	puts $fid "set computedRMX $remoteWorkDir/[file tail $computedRMX]"
	puts $fid "set computedRMY $remoteWorkDir/[file tail $computedRMY]"
	puts $fid "set varList \" $varList \""
	puts $fid "set scriptParameters \" [array get optionsArray] \""
	puts $fid "set qsubCommand \" [ProtectDollarSigns $qsubCommand] \""
	puts $fid "set qsubRespProcCommand \" [ProtectDollarSigns $qsubRespProcCommand] \""
	#------ Have to do prefix-suffix inside remote program because the waiting in the main program is after the 
	#------ entire loop is done.
	if [string length $colPrefix] { puts $fid "set colPrefix $colPrefix" } else { puts $fid "set colPrefix \"\"" }
	if [string length $colSuffix] { puts $fid "set colPrefix $colSuffix" } else { puts $fid "set colSuffix \"\"" }
	close $fid
	
	lappend copyFiles $locoBinDir/remoteCalculateDerivative $locoBinDir/calculateResponseMatrix \
	    $locoBinDir/runCalculateOrbits $scriptName $locoBinDir/tclLib/tclIndex $definitionFile $optionsFile
	set copyFiles [concat $copyFiles [glob $locoBinDir/tclLib/*.tcl]]

	OutputStatusMessage "Starting remote job..."
	set command "exec $locoBinDir/remoteCalculateDerivative -workDir $workDir -remotePath $remotePath -optionsFile $optionsFile \
	    -mode copy -matrixFile $matrixFile -copyFiles \"$copyFiles\" -doneFile $doneFile -abortFile $matrixAbortFile \
	    -errorFile $errorFile -verbose $verbose"
	OutputStatusMessage $command -logFileOnly 1
	eval $command &
	#------ If you submit 2 jobs (quad and skew), they might get errors without pause
	after 1000
    }
    #------ Waiting for the doneFile if required...
    if $waitForCompletion {
	while {[file exists $doneFile] != 1} {
	    after 1000
	    update
	    if [file exists $abortFile] {
		exec echo ABORT > $matrixAbortFile
		return -code error "Interrupted by user."
	    }
	}
    }
}

#-------------------------------------------------------------------------------------------------------------------------
proc ProtectDollarSigns {string} {
    set newString ""
    set stringLength [string length $string]
    for {set i 0} {$i < $stringLength} {incr i} {
	if {[string compare [string range $string $i $i] "$"] == 0} {
	    append newString "\\\$"
	} else {
	    append newString [string range $string $i $i]
	}
    }
    return $newString
}
#-------------------------------------------------------------------------------------------------------------------------
proc ProtectQuoteSigns {string} {
    set newString ""
    set stringLength [string length $string]
    for {set i 0} {$i < $stringLength} {incr i} {
	if {[string compare [string range $string $i $i] "\""] == 0} {
	    append newString "\\\""
	} else {
	    append newString [string range $string $i $i]
	}
    }
    return $newString
}
#-------------------------------------------------------------------------------------------------------------------------
#------ This procedure first creates matrix with number of columns equal to that in the RM file. Not used (if any) columns
#------ are removed at the very end.

proc QuickRMDeriv {args} {
    
    global tmpDir locoBinDir options hrmMeasured vrmMeasured dispColumn env verbose
    
    APSParseArguments {mode plane rmFile variableFile outputFile coupledMatrix}
    
    set tmpRoot $tmpDir/[APSTmpString]-quickRMD
    
    set addRows2hor 0
    set addRows2ver 0
    if [catch {Fit_DeleteElementsFromList -elementList [join [exec sddsquery -col $hrmMeasured] ] \
                   -deleteElements "BPMName $dispColumn"} hCorrListMeasured] {
        return -code error "Fit_DeleteElementsFromList: $hCorrListMeasured"
    }
    if [catch {Fit_DeleteElementsFromList -elementList [join [exec sddsquery -col $vrmMeasured] ] \
                   -deleteElements "BPMName $dispColumn"} vCorrListMeasured] {
        return -code error "Fit_DeleteElementsFromList: $vCorrListMeasured"
    }
    #------ Here hBpm is not xBpm. It depends on the file hrmMeasured (could be yBpm for coupledMatrix=1)
    set hBpmListMeasured [join [exec sdds2stream $hrmMeasured -col=BPMName] ]
    set vBpmListMeasured [join [exec sdds2stream $vrmMeasured -col=BPMName] ]
    set xCorrN [llength $hCorrListMeasured]
    set yCorrN [llength $vCorrListMeasured]
    set hBpmsN [llength $hBpmListMeasured]
    set vBpmsN [llength $vBpmListMeasured]
    if $options(useTune) {set addTuneRows 2} else {set addTuneRows 0}
    if $options(fitDispersion) {set dispCol 1} else {set dispCol 0}
    switch -exact -- $coupledMatrix {
        0 {
            switch -exact -- $plane {
                X {
                    set addZeroRowsBefore 0
                    set addZeroRowsAfter [expr $yCorrN * $vBpmsN + $addTuneRows]
                    set corNumberMeasured $xCorrN
                    set bpmNumberMeasured $hBpmsN
                }
                Y {
                    set addZeroRowsBefore [expr ($xCorrN + $dispCol) * $hBpmsN]
                    set addZeroRowsAfter $addTuneRows
                    set corNumberMeasured $yCorrN
                    set bpmNumberMeasured $vBpmsN
                }
            }
        }
        1 {
            switch -exact -- $plane {
                X {
                    set addZeroRowsBefore 0
                    set addZeroRowsAfter [expr $yCorrN * $vBpmsN + $addTuneRows]
                    set corNumberMeasured $xCorrN
                    set bpmNumberMeasured $hBpmsN
                }
                Y {
                    set addZeroRowsBefore [expr ($xCorrN + $dispCol) * $hBpmsN]
                    set addZeroRowsAfter $addTuneRows
                    set corNumberMeasured $yCorrN
                    set bpmNumberMeasured $vBpmsN
                }
            }
        }
        2 {
            set bpmNumberMeasured $hBpmsN
            switch -exact -- $plane {
                X {
		    if {[string compare $mode "bpmTilt"] == 0} {
			set addZeroRowsBefore [expr ($xCorrN + $dispCol) * $bpmNumberMeasured]
			set addZeroRowsAfter $addTuneRows
		    } else {
			set addZeroRowsBefore 0
			set addZeroRowsAfter [expr $yCorrN * $bpmNumberMeasured + $addTuneRows]
		    }
                }
                Y {
		    if {[string compare $mode "bpmTilt"] == 0} {
			set addZeroRowsBefore 0
			set addZeroRowsAfter [expr $yCorrN * $bpmNumberMeasured + $addTuneRows]
		    } else {
			set addZeroRowsBefore [expr ($xCorrN + $dispCol) * $bpmNumberMeasured]
			set addZeroRowsAfter $addTuneRows
		    }
                }
            }
        }
    }
    
    #------ Compare number correctors and bpms in calculated anf measured files (done to avoid possible troubles 
    #------ with c program rmDerivative which would be much more difficult to understand).
    if [catch {llength [exec sddscombine $rmFile -pipe=out -merge \
			    | sdds2stream -pipe=in -col=BPMName]} bpmNumberCalculated] {
	return -code error "Error reading BPMName column from rmFile $rmFile (1): $bpmNumberCalculated"
    }
    if [catch {llength [Fit_DeleteElementsFromList -elementList [join [exec sddsquery -col $rmFile] ] \
			    -deleteElements "BPMName $dispColumn"]} corNumberCalculated] {
        return -code error "Fit_DeleteElementsFromList: $corNumberCalculated"
    }

    #------ Getting list of varying elements for later comparison with existing columns (to skip filtering if
    #------ they are the same).
    if ![string compare $mode "cor"] {
	if [catch {Fit_DeleteElementsFromList -elementList [join [exec sddsquery -col $rmFile] ] \
		       -deleteElements BPMName} rmList] {
	    return -code error "Fit_DeleteElementsFromList: $rmList"
	}
    } else {
        if [catch {join [exec sddscombine $rmFile -pipe=out -merge \
			     | sdds2stream -pipe=in -col=BPMName] } rmList] {
	    return -code error "Error reading BPMName column from rmFile $rmFile (2): $rmList"
	}
    }

    #------ Running rmDerivative...
    if {[string compare $mode "bpmTilt"] == 0} {set localMode bpm} else {set localMode $mode}
    OutputStatusMessage "Running rmDerivative: sddscombine $rmFile -pipe=out -merge \
		   | $locoBinDir/rmDerivative.$env(HOST_ARCH) -pipe=in $outputFile -mode=$localMode \
		   -addRowsBefore=$addZeroRowsBefore -addRowsAfter=$addZeroRowsAfter" \
	-logFileOnly 1

    if [catch {exec sddscombine $rmFile -pipe=out -merge \
		   | $locoBinDir/rmDerivative.$env(HOST_ARCH) -pipe=in $outputFile -mode=$localMode \
		   -addRowsBefore=$addZeroRowsBefore -addRowsAfter=$addZeroRowsAfter} result] {
	OutputStatusMessage "Command giving error: sddscombine $rmFile -pipe=out -merge \
		   | $locoBinDir/rmDerivative.$env(HOST_ARCH) -pipe=in $outputFile -mode=$localMode \
		   -addRowsBefore=$addZeroRowsBefore -addRowsAfter=$addZeroRowsAfter"
        return -code error "Error running rmDerivative: $result"
    }
    OutputStatusMessage "rmDerivative is done." -logFileOnly 1

    #------ Now add appendix X or Y to bpm names. This part is going to be obsolete when bpm names are changed to unique.
    set appendix ""
    if ![string compare $mode "bpm"] {
        switch -exact -- $coupledMatrix {
            0 {
                switch -exact -- $plane {
                    X { set appendix1 X }
                    Y { set appendix1 Y }
                }
            }
            1 {
                switch -exact -- $plane {
                    X { set appendix1 Y }
                    Y { set appendix1 X }
                }
            }
            2 {
                #------ Ignore it here, will be error when bpms have not unique names.
                set appendix1 ""
            }
        }
        append appendix $appendix1
    }

    #------ Removing not used variables...
    set varList [join [exec sdds2stream $variableFile -col=ElementName] ]
    if [catch {RemoveNotUsedColumns -varList $varList -rmList $rmList -matrixFile $outputFile} result] {
        return -code error "RemoveNotUsedColumns: $result"
    }
    OutputStatusMessage "[exec date]: End." -logFileOnly 1
}

#-------------------------------------------------------------------------------------------------------------------------
#------ If not all variables are used in the fit (but used in the RM), filter them out from the matrix file.

proc RemoveNotUsedColumns { args } {
    APSParseArguments { varList rmList matrixFile }
    if {[llength $varList] != [llength $rmList]} {
        set removeList ""
        foreach rmElement $rmList {
            if {[lsearch -exact $varList $rmElement] == -1} {
                lappend removeList $rmElement
            }
        }
        exec sddsconvert $matrixFile -nowarning -del=col,[join $removeList ,]
        file delete $matrixFile~
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateTiltedCorrResponse { args } {

    global tmpDir workDir 
    global variableFile acceleratorCode locoSearchString hrmIterFile vrmIterFile solutionParamFileList
    APSParseArguments { lteFile beamlineName latticeParamFileList optimFile coupledMatrix \
			    corrTiltXRMDSource corrTiltYRMDSource }
    
    #------ tmpRoot is local here so that nodes knew the file:
    set tmpRoot $workDir/[APSTmpString]-crossRM
    
    #------ Preparing files for calculating RM derivative with respect to corrector tilts:
    #------ We calculate response matrix where all correctors are tilted, take difference from
    #------ initial RM and then divide it by delta. This file is then used by QuickRMDeriv
    #------ to build RM derivative.

    if [catch {join [exec sdds2stream $variableFile.hCorrTilt -col=ElementName] } hCorrList] {
        return -code error "Error reading $variableFile.hCorrTilt: $hCorrList"
    }
    if [catch {join [exec sdds2stream $variableFile.vCorrTilt -col=ElementName] } vCorrList] {
        return -code error "Error reading $variableFile.vCorrTilt: $vCorrList"
    }
    set tiltParamFile $tmpRoot.tiltParam
    lappend deleteFiles $tiltParamFile
    switch -regexp -- $acceleratorCode {
        elegant {
            set delta 0.001
            exec sddsmakedataset -pipe=out -col=ElementName,type=string -data=[join [concat $hCorrList $vCorrList] ,] \
                | sddsprocess -pipe=in $tiltParamFile -print=col,ElementParameter,TILT -def=col,ParameterValue,$delta \
                -print=col,ParameterMode,absolute
	    set localParamFileList [concat $solutionParamFileList $tiltParamFile]
	    set latticeOptions "-lteFile $lteFile -beamlineName $beamlineName -latticeParamFileList \"$localParamFileList\""
        }
        optim {
	    set delta 1.0
            if [catch {open $tiltParamFile w} fid] { 
                return -code error "$fid" 
            }
            foreach corr [concat $hCorrList $vCorrList] {
                puts $fid "\$a$corr = \$a$corr + $delta ;"
            }
            close $fid
	    set crossRMLatticeFile $workDir/[file tail $latticeFile].XRM
	    lappend deleteFiles $workDir/[file tail $latticeFile].XRM
            if [catch {Fit_AddLine -inputFile $latticeFile -outputFile $crossRMLatticeFile -searchString $locoSearchString \
                           -includeString "\#include $tiltParamFile" -substitute 0} result] {
                return -code error "Fit_AddLine: $result"
            }
	    set latticeOptions "-optimFile $crossRMLatticeFile"
        }
        default {
            return -code error "Accelerator code $acceleratorCode is not known."
        }
    }
    
    if [catch {eval CalculateResponseMatrix $latticeOptions \
		   {-acceleratorCode $acceleratorCode \
			-hrmFile $tmpRoot.tiltCorrXRM -vrmFile $tmpRoot.tiltCorrYRM -hCorrList $hCorrList -vCorrList $vCorrList \
			-workDir $workDir -twiFile $tmpRoot.twi -coupledMatrix $coupledMatrix}} result] {
        return -code error "CalculateResponseMatrix: $result"
    }
    lappend deleteFiles $tmpRoot.tiltCorrXRM $tmpRoot.tiltCorrYRM $tmpRoot.twi

    #------ Filtering out not used bpms (or other elements) and calculating (M-Mo)/delta and combining in one page for rmDeriv.c...
    foreach tiltCorrFile [list $corrTiltXRMDSource $corrTiltYRMDSource] iterFile [list $hrmIterFile $vrmIterFile] \
	ext [list tiltCorrXRM tiltCorrYRM] {
	    if [catch {exec sddsselect $tmpRoot.$ext $iterFile -pipe=out -match=BPMName \
			   | sddschanges -pipe -base=$iterFile -changes=exclude=BPMName,* -copy=BPMName -parallelPages \
			   | sddsprocess -pipe "-redef=col,%s,%s $delta /,select=ChangeIn*" \
			   | sddsconvert -pipe "-edit=col,ChangeIn*,8d" \
			   | sddscombine -pipe=in $tiltCorrFile -merge -overWrite} result] {
		return -code error "Error processing corrector cross response (files: $tmpRoot.$ext and $iterFile): $result"
	    }
	}
    eval file delete $deleteFiles

}

#-------------------------------------------------------------------------------------------------------------------------

proc FilterResponseMatrixDeriv { args } {
    global tmpDir dispColumn makeSkewNamesUnique options
    set tmpRoot $tmpDir/[APSTmpString]-filtMatr
    APSParseArguments {inputFile outputFile elementListVar elementListRM optionsList coupledMatrix fitDispersion \
			   useTune useKickFiles kickFileX kickFileY}

    OutputStatusMessage "[exec date +%D--%T]: Filtering response matrix derivative (using source $inputFile)..."
    if $useKickFiles {OutputStatusMessage "Warning: for useKickFiles=1, filtering is done for columns only."}

    #------ Checking that we are not overwriting inputFile (that inputFile and outputFile are different files). 
    #------ Cannot just compare filenames because of disk mounting at APS.
    if ![file exists $inputFile] {
	return -code error "Error: File $inputFile does not exist."
    }
    if [file exists $outputFile] {
	file rename -force $outputFile $outputFile.tmpcopy
	if ![file exists $inputFile] {
	    file rename -force $outputFile.tmpcopy $outputFile
	    return -code error "Error: Input and output files are the same: $inputFile --- $outputFile"
	}
	file rename -force $outputFile.tmpcopy $outputFile
    }

    #------ Inserting $dispColumn at the end of H corr list (index 1 in the big list):
    if {$fitDispersion && $options(useCorrs)} {
	set elementListVar [lreplace $elementListVar 1 1 \
			     [linsert [lindex $elementListVar 1] [llength [lindex $elementListVar 1]] $dispColumn]]
    }

    set filterColumns 0
    set filterRows 0
    set columnListOld [exec sddsquery -col $inputFile]
    if [catch {BuildColumnList -elementList $elementListVar -coupledMatrix $coupledMatrix \
		   -makeSkewNamesUnique $makeSkewNamesUnique} columnListNew] {
	return -code error "BuildColumnList: $columnListNew"
    }
    #------ Checking for elements that are not in original derivative matrix:
    set missingElements ""
    set error 0
    foreach column $columnListNew {
	if {[lsearch -exact $columnListOld $column] == -1} {
	    set error 1
	    lappend missingElements $column
	}
    }
    if $error {return -code error "Error: Element that is not in original matrix is used for filtering: $missingElements."}
    if {[llength $columnListNew] < [llength $columnListOld]} {
	set filterColumns 1
	set removeColumnList ""
	foreach column $columnListOld {
	    if {[lsearch -exact $columnListNew $column] == -1} {
		lappend removeColumnList $column
	    }
	}
	OutputStatusMessage "Filtering: number of columns to remove: [llength $removeColumnList]" -logFileOnly 1
    }

    #------ Building lists of new and old rows:
    set rowListOld [exec sdds2stream $inputFile -col=Rootname]
    if [catch {BuildRowList -elementList $elementListRM -coupledMatrix $coupledMatrix \
		   -fitDispersion $fitDispersion -useTune $useTune} rowListNew] {
	return -code error "BuildRowList: $rowListNew"
    }

    #------ Building file with rows to be removed in the Rootname column:
    set absentRows ""
    if {[llength $rowListNew] < [llength $rowListOld]} {
	set filterRows 1
	foreach element $rowListOld {
	    if {[lsearch -exact $rowListNew $element] == -1} { lappend absentRows $element }
	}
	set rowLimit 7000
	if {[llength $absentRows] > $rowLimit} {
	    set splits [expr [llength $absentRows] / $rowLimit + 1]
	    set newList [Fit_SplitList -List $absentRows -splitTasks $splits]
	    set fileList ""; set counter 0
	    foreach smallList $newList {
		if [catch {exec sddsmakedataset $tmpRoot.$counter -col=Rootname,type=string -data=[join $smallList ,]} result] {
		    return -code error "Error running sddsmakedataset (1) (list length [llength $smallList]): $result"
		}
		lappend fileList $tmpRoot.$counter
		incr counter
	    }
	    eval exec sddscombine $fileList $tmpRoot -merge
	    eval file delete $fileList
	} else {
	    if [catch {exec sddsmakedataset $tmpRoot -col=Rootname,type=string -data=[join $absentRows ,]} result] {
		return -code error "Error running sddsmakedataset (2) (list length [llength $absentRows]): $result"
	    }
	}
    }
    OutputStatusMessage "Filtering: Input file is $inputFile; output file is $outputFile. Old rows: [llength $rowListOld]; new rows:[llength $rowListNew]; absent rows: [llength $absentRows]." -logFileOnly 1

    set tmpOutputFile $tmpDir/[file tail $outputFile]
    file delete $tmpOutputFile
    #------ Do not filter rows in useKickFiles=1 (don't have time to do it).
    if $useKickFiles {set filterRows 0}
    #------ Applying filtering commands:
    if {$filterColumns && !$filterRows} {
	exec sddsconvert $inputFile $tmpOutputFile -del=col,[join $removeColumnList ,]
    }
    if {$filterColumns && $filterRows} {
	exec sddsconvert $inputFile -pipe=out -del=col,[join $removeColumnList ,] \
	    | sddsselect -pipe=in $tmpRoot $tmpOutputFile -match=Rootname -invert
    }
    if {!$filterColumns && $filterRows} {
	exec sddsselect $inputFile $tmpRoot $tmpOutputFile -match=Rootname -invert
    }
    file delete $tmpRoot
    if {$filterColumns || $filterRows} {
	OutputStatusMessage "[exec date +%D--%T]: Filtering is done."
    } else {
	OutputStatusMessage "[exec date +%D--%T]: $inputFile has no columns to filter. Copying..."
	file copy -force $inputFile $tmpOutputFile
    }
    file copy -force $tmpOutputFile $outputFile
    file delete $tmpOutputFile
}

#-------------------------------------------------------------------------------------------------------------------------

proc BuildColumnList { args } {
    APSParseArguments { elementList coupledMatrix makeSkewNamesUnique}
    #------ Adding X or Y to bpm names (only when coupledMatrix is 0 or 1):
    switch -exact -- $coupledMatrix {
	0 {
	    foreach index [list 3 4] suffix [list X Y] {
		set elementList [lreplace $elementList $index $index \
				     [EditListElements -elementList [lindex $elementList $index] -suffix $suffix]]
	    }
	}
	1 {
	    foreach bpmIndex [list 3 4] plane [list Y X] {
		set elementList [lreplace $elementList $index $index \
				     [EditListElements -elementList [lindex $elementList $index] -suffix $suffix]]
	    }
	}
	2 {
	    #------ Adding prefix "a" to corrector tilts (NO BPMS NOW!!!!!!!!!!!!!!!!!!!!):
	    foreach index [list 7 8 9 10] prefix [list a a a a] {
		set elementList [lreplace $elementList $index $index \
				     [EditListElements -elementList [lindex $elementList $index] -prefix $prefix]]
	    }
	}
    }
    #------ Adding E to corrector names for energy fitting:
    set index 5
    set suffix E
    set elementList [lreplace $elementList $index $index \
			 [EditListElements -elementList [lindex $elementList $index] -suffix $suffix]]
    #------ Adding a before skew quad names:
    if $makeSkewNamesUnique {
	set index 6
	set prefix a
	set elementList [lreplace $elementList $index $index \
			     [EditListElements -elementList [lindex $elementList $index] -prefix $prefix]]
    }
    return [concat Rootname [join $elementList ]]
}

#-------------------------------------------------------------------------------------------------------------------------

proc EditListElements { args } {
    set prefix ""
    set suffix ""
    APSParseArguments {elementList prefix suffix}
    set localList ""
    foreach element $elementList {
	lappend localList ${prefix}${element}${suffix}
    }
    return $localList
}

#-------------------------------------------------------------------------------------------------------------------------

proc BuildRowList { args } {
    global dispColumn
    APSParseArguments {elementList coupledMatrix fitDispersion useTune}
    #------ Inserting $dispColumn at the end of the H corr list (position 0 in the big list):
    if $fitDispersion {
	set elementList [lreplace $elementList 0 0 \
			     [linsert [lindex $elementList 0] [llength [lindex $elementList 0]] $dispColumn]]
    }
    switch -exact -- $coupledMatrix {
	0 {
	    set rowList ""
	    foreach bpmIndex [list 2 3] corrIndex [list 0 1] {
		foreach bpm [lindex $elementList $bpmIndex] {
		    foreach corr [lindex $elementList $corrIndex] {
			lappend rowList $corr$bpm
		    }
		}
	    }
	}
	1 {
	    set rowList ""
	    foreach bpmIndex [list 3 2] corrIndex [list 0 1] {
		foreach bpm [lindex $elementList $bpmIndex] {
		    foreach corr [lindex $elementList $corrIndex] {
			lappend rowList $corr$bpm
		    }
		}
	    }
	}
	2 {
	    set rowList ""
	    foreach corrIndex [list 0 1] {
		foreach bpm [concat [lindex $elementList 2] [lindex $elementList 3]] {
		    foreach corr [lindex $elementList $corrIndex] {
			lappend rowList $corr$bpm
		    }
		}
	    }
	}
    }
    if $useTune {
	set rowList [concat $rowList Snux Snuy]
    }
    return $rowList
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateSVDdecomposition { args } {

    APSParseArguments { matrixFile programName }

    global tmpDir matlabPipeFile matlabStarted matlabErrorFile matlabScriptDir
    
    OutputStatusMessage "Doing SVD of file $matrixFile using $programName..."
    #------ Deleting all parameters and a character column from the matrix file.
    exec sddsconvert $matrixFile $matrixFile.tmp -del=par,* -del=col,Rootname
    file delete $matrixFile.U $matrixFile.V $matrixFile.S $matrixFile.SV

    switch -exact -- $programName {
	matlab {
	    if !$matlabStarted {
		if [catch { StartMatlabInterface } matlabStarted] {
		    return -code error "StartMatlabInterface: $matlabStarted"
		}
	    }
	    if $matlabStarted {
		set matlabCommand "FILE_NAME=\'$matrixFile.tmp\'; "
		append matlabCommand "FILE_NAME_U=\'$matrixFile.U\'; "
		append matlabCommand "FILE_NAME_V=\'$matrixFile.V\'; "
		append matlabCommand "FILE_NAME_S=\'$matrixFile.S\'; "
		append matlabCommand "FILE_NAME_SV=\'$matrixFile.SV\'; "
		append matlabCommand "matlabSVD; "
		exec echo $matlabCommand > $matlabPipeFile
		while {[file exists $matlabPipeFile] == 1} { 
		    after 2000 
		    update
		    if [file exists $matlabErrorFile] {
			return -code error "Matlab returned error: check $matlabErrorFile file"
		    }
		}
	    } 
	}
	octave {
	    set tmpRoot $tmpDir/[APSTmpString]-octave
            if [catch {open $tmpRoot.m w} fid] { 
                return -code error "$fid" 
            }
	    puts $fid "LOADPATH=\":$matlabScriptDir\";"
	    puts $fid "FILE_NAME=\'$matrixFile.tmp\'; "
	    puts $fid "FILE_NAME_U=\'$matrixFile.U\'; "
	    puts $fid "FILE_NAME_V=\'$matrixFile.V\'; "
	    puts $fid "FILE_NAME_S=\'$matrixFile.S\'; "
	    puts $fid "FILE_NAME_SV=\'$matrixFile.SV\'; "
	    puts $fid "matlabSVD; "
	    close $fid
	    if [catch {exec octave -q $tmpRoot.m} result] {
		if {[string first "warning:" $result] == -1} {
			return -code error "Error running octave: $result"
		}
	    }
	    file delete $tmpRoot.m
	}
    }
    exec sddsprocess $matrixFile.SV -nowarning -def=col,Index,i_row
    file delete $matrixFile.tmp $matrixFile.SV~
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateRMDerivInverse { args } {

    APSParseArguments { SVnumber rmDerivFile rmDerivInverseFile rmDifferenceFile programName }

    global matlabPipeFile matlabStarted matlabErrorFile tmpDir matlabScriptDir
    
    OutputStatusMessage "[exec date +%D--%T]: Building the inverse matrix using $programName..."
    catch {eval file delete [glob ${rmDerivInverseFile}*]}
    switch -exact -- $programName {
	matlab {
	    if !$matlabStarted {
		if [catch { StartMatlabInterface } matlabStarted] {
		    return -code error "StartMatlabInterface: $matlabStarted"
		}
	    }
            exec sddstranspose $rmDifferenceFile -pipe=out -newColumnNames=Rootname \
                | sddsconvert -pipe=in $rmDerivInverseFile -del=par,* -del=col,OldColumnNames -removePages=1
            set matlabCommand    "FILE_NAME_UT=\'$rmDerivFile.U\'; "
            append matlabCommand "FILE_NAME_VT=\'$rmDerivFile.V\'; "
            append matlabCommand "FILE_NAME_S= \'$rmDerivFile.S\'; "
            append matlabCommand "OUTPUT_FILE_NAME=\'$rmDerivInverseFile\'; "
            append matlabCommand "NUMBER_OF_SV=$SVnumber; "
            append matlabCommand "matlabBuildInverse; "
            exec echo $matlabCommand > $matlabPipeFile
            while {[file exists $matlabPipeFile] == 1} { 
                after 2000 
                update 
                if [file exists $matlabErrorFile] {
                    return -code error "Matlab returned error: check $matlabErrorFile file"
                }
            }
	}
	octave {
            exec sddstranspose $rmDifferenceFile -pipe=out -newColumnNames=Rootname \
                | sddsconvert -pipe=in $rmDerivInverseFile -del=par,* -del=col,OldColumnNames -removePages=1
	    set tmpRoot $tmpDir/[APSTmpString]-octave
            if [catch {open $tmpRoot.m w} fid] { 
                return -code error "$fid" 
            }
	    puts $fid "LOADPATH=\":$matlabScriptDir\";"
            puts $fid "FILE_NAME_UT=\'$rmDerivFile.U\'; "
            puts $fid "FILE_NAME_VT=\'$rmDerivFile.V\'; "
            puts $fid "FILE_NAME_S= \'$rmDerivFile.S\'; "
            puts $fid "OUTPUT_FILE_NAME=\'$rmDerivInverseFile\'; "
            puts $fid "NUMBER_OF_SV=$SVnumber; "
            puts $fid "matlabBuildInverse; "
	    close $fid
	    if [catch {exec octave -q $tmpRoot.m} result] {
		if {[string first "warning:" $result] == -1} {
		    return -code error "Error running octave: $result"
		}
	    }
	    file delete $tmpRoot.m
	}
	sdds {
	    global svdProgramFullName
	    if [info exists svdProgramFullName] {
		set executable $svdProgramFullName
	    } else {
		set executable sddspseudoinverse
	    }
	    OutputStatusMessage "Using the following file for sddspseudoinverse: $executable" -logFileOnly 1
	    if [catch {exec $executable $rmDerivFile -pipe=out -largestSingularValues=$SVnumber \
			   -sFile=$rmDerivFile.SV -economy \
			   | sddsconvert -pipe=in $rmDerivInverseFile -del=col,OldColumnNames -del=para,*} result] {
		return -code error "Error doing sddspseudoinverse: $result"
	    }
            exec sddsconvert $rmDerivFile.SV -nowarning -rename=col,SingularValues=SV
            file delete $rmDerivFile.SV~
	}
    }
    file stat $rmDerivInverseFile statArray
    OutputStatusMessage "RM Inverse calculated. The size is $statArray(size)"

}

#-------------------------------------------------------------------------------------------------------------------------

proc LoadRMInverse {args} {
    
    APSParseArguments { workDir rmDerivInverseFile recalculateInverse }
    global matlabStarted matlabPipeFile matlabErrorFile
    
    if !$matlabStarted {
        if [catch { StartMatlabInterface } matlabStarted] {
            return -code error "StartMatlabInterface: $matlabStarted"
        }
    }
    if $matlabStarted {
        if $recalculateInverse {
            set matlabCommand "RM=A'; clear A; "
        } else {
            set matlabCommand "FILE_NAME=\'$rmDerivInverseFile\'; matlabLoadRM; "
        }
        append matlabCommand "FILE_NAME=\'$workDir/matlab.xycolumn\'; FILE_NAME1=\'$workDir/matlab.xycolumn.tmp\'; "
        exec echo $matlabCommand > $matlabPipeFile
        while {[file exists $matlabPipeFile] == 1} { 
            after 1000 
            update
            if [file exists $matlabErrorFile] {
                return -code error "Matlab returned error: check $matlabErrorFile file"
            }
        }
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc StartMatlabInterface { args } {

    global matlabScriptDir matlabPipeFile matlabLogFile matlabErrorFile workDir

    set waitInterval 2000
    set nIntervals 40
    set matlabPipeFile $workDir/matlabPipe.txt
    set matlabLogFile $workDir/matlabSession.log
    set matlabErrorFile $workDir/matlabError.log
    # killing all matlabInterface processes if any by printing Exit to matlabPipeFile
    if [file exists $matlabPipeFile] {file delete $matlabPipeFile}
    while {![file exists $matlabPipeFile]} {
        exec echo Exit > $matlabPipeFile
        after $waitInterval
    }
    
    file delete $matlabPipeFile $matlabLogFile $matlabErrorFile
    if [catch {exec $matlabScriptDir/matlabInterface -logFile $matlabLogFile -pipeFile $matlabPipeFile \
                   -errorFile $matlabErrorFile &} result] {
        return -code error "Error starting matlab interface: $result"
    }
    exec echo "addpath(\'$matlabScriptDir\');" > $matlabPipeFile
    set icount 0
    set matlabStarted 1
    while {[file exists $matlabPipeFile] == 1} { 
        after $waitInterval; update; incr icount
        if {$icount > $nIntervals} {
            if [catch {APSMultipleChoice .matlabChoice -question "Starting matlab takes too long. What do you want to do?" \
                           -labelList [list "Continue waiting\nfor matlab to start" "Continue without\nmatlab" \
                                           "Stop\ncalculations"] \
                           -returnList [list 0 1 2]} matlabChoice] {
                return -code error "APSMultipleChoice: $matlabChoice"
            }
            switch -exact -- $matlabChoice {
                0 { set icount 0 }
                1 { set matlabStarted 0
                    file delete $matlabPipeFile }
                2 { return -code error "Cannot start matlab engine. Interrupted by user." }
            }
        }
    }
    return $matlabStarted
}

#-------------------------------------------------------------------------------------------------------------------------

proc StopMatlabInterface { } {
    global matlabStarted matlabPipeFile
    if [info exists matlabStarted] {
        if $matlabStarted {
            if [file exists $matlabPipeFile] {file delete $matlabPipeFile}
            while {![file exists $matlabPipeFile]} {
                exec echo Exit > $matlabPipeFile
                after 2000
            }
            file delete $matlabPipeFile
        }
    }
}

#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
#------ Miscelaneous procedures used during iterations like output, model update, etc. --------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------

proc IterationResultsOutput {args} {

    global useQuadConstraints
    set vectorStdDev ""
    APSParseArguments {rmsList fitDispersion vectorStdDev coupledMatrix iteration sddsFile secondRun}

    set outputLine ""
    set lineVectorStdDev ""
    array set rmsArray $rmsList
    array set sddsArray [list iteration $iteration stDev 0.0 rmsTotal 0.0 rmsTotalWQ 0.0 rmsDX 0.0 rmsDY 0.0 \
			     rmsXX 0.0 rmsXY 0.0 rmsYX 0.0 rmsYY 0.0 rmsQ 0.0]

    foreach index [array names rmsArray] {set sddsArray($index) $rmsArray($index)}

    if [string length $vectorStdDev] {
        set lineVectorStdDev [format "%s %9.3e" "Var. stdev:" $vectorStdDev]
	if $useQuadConstraints {append lineVectorStdDev [format "%s %9.3e " ". Total res. error: " $rmsArray(rmsTotal)]}
        append lineVectorStdDev "\n"
	set sddsArray(stDev) $vectorStdDev
    }

    if {$coupledMatrix == 2} {
        set lineX [format "%9.3e,%9.3e" $rmsArray(rmsXX) $rmsArray(rmsXY)]
        set lineY [format "%9.3e,%9.3e" $rmsArray(rmsYX) $rmsArray(rmsYY)]
    } else {
        set lineX [format "%9.3e" $rmsArray(rmsXX)]
        set lineY [format "%9.3e" $rmsArray(rmsYY)]
    }
    if $fitDispersion {
        if {$coupledMatrix == 2} {
            set lineDisp "D(m): [format "%9.3e,%9.3e" $rmsArray(rmsDX) $rmsArray(rmsDY)])"
        } else {
            set lineDisp "D(m): [format "%9.3e" $rmsArray(rmsDX)])"
        }
    } else {
        set lineDisp ")"
    }
    if $useQuadConstraints {
	set lineQuadConstraints " Quads: [format "%9.3e" $rmsArray(rmsQ)]"
	set totalRmsOutput $rmsArray(rmsTotalWQ)
    } else {
	set lineQuadConstraints ""
	set totalRmsOutput $rmsArray(rmsTotal)
    }
    set lineRes [format "%s %9.3e %s %s %s %s " "Res. error: " $totalRmsOutput " (X(mm):" $lineX "Y(mm):" $lineY]
    append outputLine $lineVectorStdDev $lineRes $lineDisp $lineQuadConstraints
    
    OutputStatusMessage $outputLine

    if [catch {OutputSddsFile -sddsList [array get sddsArray] -outputFile $sddsFile -secondRun $secondRun} result] {
	return -code error "OutputSddsFile: $result"
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc OutputSddsFile {args} {
    APSParseArguments {sddsList outputFile secondRun}
    array set sddsArray $sddsList
    set deleteFiles ""
    if {$secondRun == 2} {
	if [catch {exec sddsconvert $outputFile -pipe=out -keep=1 \
		       | tee $outputFile.01 \
		       | sddsprocess -pipe -clip=0,1,inv \
		       | sdds2stream -pipe=out -col=Iteration} iterations0] {
	    return -code error "Error reading iterations0: $iterations0"
	}
 	if {$sddsArray(iteration) != 0} {
	    exec sddsconvert $outputFile $outputFile.02 -keep=2
	}
	set origOutputFile $outputFile
	set outputFile $outputFile.02
	lappend deleteFiles $origOutputFile.01 $origOutputFile.02
	set outputIter [expr $sddsArray(iteration) + $iterations0]
    } else {
	set outputIter $sddsArray(iteration)
    }
    exec sddsmakedataset $outputFile.tmp \
	-col=Iteration,type=double -data=$outputIter \
	-col=StDev,type=double -data=$sddsArray(stDev) \
	-col=TotalRms,type=double -data=$sddsArray(rmsTotal) \
	-col=TotalRmsWQ,type=double -data=$sddsArray(rmsTotalWQ) \
	-col=RmsXX,type=double -data=$sddsArray(rmsXX) \
	-col=RmsXY,type=double -data=$sddsArray(rmsXY) \
	-col=RmsYX,type=double -data=$sddsArray(rmsYX) \
	-col=RmsYY,type=double -data=$sddsArray(rmsYY) \
	-col=RmsDx,type=double -data=$sddsArray(rmsDX) \
	-col=RmsDy,type=double -data=$sddsArray(rmsDY) \
	-col=RmsQ,type=double -data=$sddsArray(rmsQ)
    if {$sddsArray(iteration) == 0} {
	file copy -force $outputFile.tmp $outputFile
	lappend deleteFiles $outputFile.tmp
    } else {
	exec sddscombine $outputFile $outputFile.tmp $outputFile.tmp1 -merge -overWrite
	file copy -force $outputFile.tmp1 $outputFile
	lappend deleteFiles $outputFile.tmp $outputFile.tmp1
    }
    if {$secondRun == 2} {exec sddscombine $origOutputFile.01 $origOutputFile.02 $origOutputFile -overWrite}
    eval file delete $deleteFiles
}

#-------------------------------------------------------------------------------------------------------------------------

proc UpdateModel {args} {
    
    APSParseArguments { elementsUpdateFile vectorOutFile corFraction variableFile allElementsFile }
    
    global workDir tmpDir options dispColumn acceleratorCode specialParamFile
    global tevQuadsIterOptimFile indexList variableTypeList adjustAverageGains
    
    set tmpRoot $tmpDir/[APSTmpString]-updateModel
    set deleteFiles ""
    #------ Preparing update file...
    if [catch { exec sddsxref $elementsUpdateFile $vectorOutFile -pipe=out -take=MatrixDifference \
                    | sddsconvert -pipe -del=para,PageID \
                    | sddsbreak -pipe -changeof=VariableType \
                    | sddsprocess -pipe -process=VariableType,first,PageID \
                    | sddsconvert -pipe=in $tmpRoot.Update -del=col,ParameterValue \
                    -rename=col,MatrixDifference=VarAdjustment} result] {
        return -code error "Error creating update file ($elementsUpdateFile; $vectorOutFile): $result"
    }
    set existingVarList [join [exec sdds2stream $tmpRoot.Update -para=PageID] ]
    #------ Adjusting average gains...
    if {[info exists adjustAverageGains] && $options(useBPMs)} {
        if [catch {AdjustAverageGains -updateFile $tmpRoot.Update -adjustAverageGains $adjustAverageGains \
                       -dispColumn $dispColumn -dataColumn VarAdjustment} result] {
            return -code error "AdjustAverageGains: $result"
        }
    }
    #------ Update variable and allElements files...
    if {![string compare $acceleratorCode elegant] && $options(useSkew)} {
	#----- Remove K1 rows from elements.Skew before updating...
	exec sddsprocess $allElementsFile.Skew -match=col,ElementParameter=TILT -nowarning
	lappend deleteFiles $allElementsFile.Skew~
    }
    set variableFileList ""
    set allElementsFileList ""
    foreach index [linsert $indexList 100 special] variableType [linsert $variableTypeList 100 special] {
        if {[lsearch -exact $existingVarList $variableType] != -1} {
            exec sddsprocess $tmpRoot.Update $tmpRoot.$variableType -nowarning \
                -match=para,PageID=$variableType
            if ![exec sdds2stream $tmpRoot.$variableType -rows=bare] {
                OutputStatusMessage "Warning: $index option is on, but $elementsUpdateFile file has zero rows of this \
                                                    variable type."
            }
            foreach rootFile [list $variableFile $allElementsFile] {
                if [catch {exec sddsxref $rootFile.$variableType $tmpRoot.$variableType -pipe=out -nowarning \
                               -take=VarAdjustment -match=ElementName -fillIn \
                               | sddsprocess -pipe \
                               "-redefine=col,ParameterValue,ParameterValue VarAdjustment $corFraction * + " \
                               | sddsconvert -pipe=in $tmpRoot.$variableType.new -del=col,VarAdjustment } result] {
                    return -code error "Error updating allElements file for $variableType type: $result"
                }
                file copy -force $tmpRoot.$variableType.new $rootFile.$variableType
                lappend deleteFiles $tmpRoot.$variableType.new
            }
        }
        lappend deleteFiles $tmpRoot.$variableType 
        lappend variableFileList $variableFile.$variableType
        lappend allElementsFileList $allElementsFile.$variableType
    }

    #------ Final combining of updated files
    eval exec sddscombine $variableFileList $variableFile.sdds -overWrite
    eval exec sddscombine $allElementsFileList $allElementsFile.sdds -overWrite
    exec sddscombine $variableFile.sdds $elementsUpdateFile -merge -overWrite
    lappend deleteFiles ${elementsUpdateFile}~ $tmpRoot.Update
    
    #------ Handling of special variables (done after final combining to use already combined elementsUpdateFile)
    if [exec sdds2stream $variableFile.special -rows=bare] {
	set specialFile $tmpRoot.special
	lappend deleteFiles $specialFile
	exec sddsprocess $elementsUpdateFile $specialFile -match=col,VariableType=special
	if [catch {join [exec sddsbreak $specialFile -pipe=out -changeof=ElementParameter \
				      | sddsprocess -pipe -process=ElementParameter,first,SpecialType \
			     | sdds2stream -pipe=in -para=SpecialType] } specialVarList] {
	    return -code error "Error reading special var file: $specialVarList"
	}
	set specialFileList ""
	foreach specialVarType $specialVarList {
	    exec sddsprocess $specialFile $specialFile.$specialVarType -match=col,ElementParameter=$specialVarType
	    lappend deleteFiles $specialFile.$specialVarType
	    set specialElementList [join [exec sdds2stream $specialFile.$specialVarType -col=ElementName] ]
	    set specialParameterValues [join [exec sdds2stream $specialFile.$specialVarType -col=ParameterValue] ]
	    set parameterMode [exec sddsprocess $specialFile.$specialVarType -pipe=out -clip=1,0 \
				   | sdds2stream -pipe=in -col=ParameterMode]
	    file delete $specialFile.$specialVarType
	    if [catch {HandleSpecialVariables -elementList $specialElementList -valueList $specialParameterValues \
			   -parameterMode $parameterMode -varType $specialVarType \
			   -outputFile $specialFile.$specialVarType} result] {
		return -code error "HandleSpecialVariables: $result"
	    }
	    lappend specialFileList $specialFile.$specialVarType
	}
	if {[llength $specialFileList] == 1} {
	    file copy -force $specialFileList $specialParamFile
	} else {
	    eval exec sddscombine $specialFileList $specialParamFile -overWrite
	}
    }

    switch -regexp -- $acceleratorCode {
        elegant {
        }
        optim {
            # Updating "Quadrupole corrector parameters" file tevQuadsIterFile.
            if {$options(useQuads) || $options(useSkew)} {
                exec sddscombine $allElementsFile.Quad $allElementsFile.Skew $tmpRoot.tev -merge -overWrite
                if [catch {TransformSddsQuads2Optim -sddsFile $tmpRoot.tev -optimFile $tevQuadsIterOptimFile} result] {
                    return -code error "TransformSddsQuads2Optim: $result"
                }
                file delete $tmpRoot.tev
            }
        }
        default {
            return -code error "Accelerator code $acceleratorCode is unknown."
        }
    }
    eval file delete $deleteFiles
}

#-------------------------------------------------------------------------------------------------------------------------
proc HandleSpecialVariables {args} {
    global tmpDir
    APSParseArguments {elementList valueList parameterMode varType outputFile}
    set tmpRoot $tmpDir/[APSTmpString]-specUpdate
    set fileList ""
    foreach elementName $elementList value $valueList {
	set filename $tmpRoot.$elementName
	lappend fileList $filename
	switch -exact $varType {
	    shift {
		#--- Shift of elements by using 2 drifts (has to be in .lte file already) with positive and negative lengths
		set elementName1 ${elementName}_D1
		set elementName2 ${elementName}_D2
		set value1 $value
		set value2 [expr $value * -1.0]
		exec sddsmakedataset $filename \
		    -col=ElementName,type=string -data=$elementName1,$elementName2 \
		    -col=ElementParameter,type=string -data=L,L \
		    -col=ParameterMode,type=string -data=$parameterMode,$parameterMode \
		    -col=ParameterValue,type=double -data=$value1,$value2
	    }
	    default {return -code error "Wrong special variable type: $varType"}
	}
    }
    if {[llength $fileList] == 1} {
	file copy -force $fileList $outputFile
    } else {
	eval exec sddscombine $fileList $outputFile -merge
    }
    eval file delete $fileList
}
#-------------------------------------------------------------------------------------------------------------------------

proc AdjustAverageGains { args } {
    APSParseArguments { updateFile adjustAverageGains dispColumn dataColumn}
    set pageIDList [join [exec sdds2stream $updateFile -para=PageID] ]
    set nSigma 3
    set averageList [join [exec sddsoutlier $updateFile -pipe=out -col=$dataColumn -stDev=$nSigma \
			       | sddsprocess -pipe -process=$dataColumn,average,Aver \
			       | sdds2stream -pipe=in -para=Aver] ]
    
    set udjNameList [list hCorr xBpm vCorr yBpm]
    switch -exact -- $adjustAverageGains {
        dispersion {
	    if [catch {exec sddsprocess $updateFile -pipe=out -match=para,PageID=hCorr \
			   | sddsprocess -pipe -match=col,ElementName=$dispColumn \
			   | sdds2stream -pipe=in -col=$dataColumn} gainAdjustmentX] {
		return -code error "Error getting dispersion gain calibration: $gainAdjustmentX"
	    }
	    if ![string length $gainAdjustmentX] {set gainAdjustmentX 0}
	    set gainAdjustmentY [lindex $averageList [lsearch -exact $pageIDList vCorr]]
	    set gainList [list [expr $gainAdjustmentX * -1] $gainAdjustmentX [expr $gainAdjustmentY * -1] $gainAdjustmentY]
	}
	averageBpm {
	    set gainAdjustmentX [lindex $averageList [lsearch -exact $pageIDList xBpm]]
	    set gainAdjustmentY [lindex $averageList [lsearch -exact $pageIDList yBpm]]
	    set gainList [list $gainAdjustmentX  [expr $gainAdjustmentX * -1] $gainAdjustmentY [expr $gainAdjustmentY * -1]]
	}
	averageCor {
	    set gainAdjustmentX [lindex $averageList [lsearch -exact $pageIDList hCorr]]
	    set gainAdjustmentY [lindex $averageList [lsearch -exact $pageIDList vCorr]]
	    set gainList [list [expr $gainAdjustmentX * -1] $gainAdjustmentX [expr $gainAdjustmentY * -1] $gainAdjustmentY]
	}
	default {
            set gainList [list 0 0 0 0]
	}
    }
    foreach pageID $pageIDList {
	set index [lsearch -exact $udjNameList $pageID]
	if {$index != -1} { lappend dataList [lindex $gainList $index] } else { lappend dataList 0 }
    }
    exec sddsmakedataset -pipe=out -col=GainAdj,type=double -data=[join $dataList ,] \
	| sddsbreak -pipe -rowLimit=1 \
	| sddsprocess -pipe=in $updateFile.gainAdj -process=GainAdj,first,GainAdjustment
    exec sddsxref $updateFile $updateFile.gainAdj -pipe=out -leave=* -transfer=para,GainAdjustment \
	| sddsprocess -pipe=in $updateFile.tmp "-redef=col,$dataColumn,$dataColumn GainAdjustment +"
    file copy -force $updateFile.tmp $updateFile
    file delete $updateFile.gainAdj $updateFile.tmp
}

#-------------------------------------------------------------------------------------------------------------------------

proc StopExecution {args} {
    
    global workDir badPointsLevel svdProgramName
    global elementOrderFile rmDerivInverseFile
    APSParseArguments {resultsFile allElementsFile hrmIterFile vrmIterFile rmsList badPointsIterCounter}

    array set rmsArray $rmsList
    set residualError $rmsArray(rmsTotal)
    if {[lsearch -exact [array name rmsArray] rmsTotalWQ] == -1} {
	set residualErrorWQ $residualError
    } else {
	set residualErrorWQ $rmsArray(rmsTotalWQ)
    }
    if ![string compare $svdProgramName "matlab"] {
	if [catch {StopMatlabInterface} result] {
	    OutputStatusMessage "StopMatlabInterface returned error: $result"
	    OutputStatusMessage "Continuing StopExecution..."
	}
    }
    unset badPointsLevel
    exec sddsprocess $allElementsFile.sdds $resultsFile -nowarning \
	-def=para,ResidualError,$residualError -def=para,ResidualErrorWQ,$residualErrorWQ \
	-def=para,BadPointsIterCounter,$badPointsIterCounter
    file copy -force $hrmIterFile $hrmIterFile.after
    file copy -force $vrmIterFile $vrmIterFile.after
    
#    file delete $workDir/vector_out $workDir/computed.Xmatrix $workDir/computed.Ymatrix
    file delete $elementOrderFile
    catch {eval file delete [glob -nocomplain -- $workDir/*~]}
}

#-------------------------------------------------------------------------------------------------------------------------

proc OutputStatusMessage {text args} {
    global outputStatusDevice outputStatusFile iterationsLogFile
    set logFileOnly 0
    APSParseArguments {logFileOnly}
    if [catch {Fit_WriteToFile -filename $iterationsLogFile -accessMode a -line $text} result] {
	puts stdout "Error:::::"
	puts stdout "Line: $text"
	puts stdout "End of line:::"
        return -code error "Fit_WriteToFile: $result"
    }
    if $logFileOnly { return }
    switch -exact $outputStatusDevice {
        statusScreen { APSSetVarAndUpdate status $text }
        stdout { puts stdout $text }
        file {
            if {![info exists outputStatusFile] || ![string length $outputStatusFile]} {
		#------ outputStatusFile is not defined. Send output to stdout...
		puts stdout $text
                return
            }
            if [file exists $outputStatusFile] {
                set fid [open $outputStatusFile a]
                puts $fid $text
                close $fid
            } else {
                set fid [open $outputStatusFile w]
                puts $fid $text
                close $fid
            }
        }
        default { return -code error "OutputStatusMessage: wrong outputStatusDevice: $outputStatusDevice" }
    }
}

#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------
