#!/bin/sh
# \
exec oagtclsh "$0" "$@"

set usage "locoGenerateResponseWithErrors -machineName <APS|TAA> -lteFile <filename> -beamline <beamline-name> \
   -paramFileList <file-list> -errorList <number-list> -definitionFile <filename> \
   -elementListFile <filename> -rootname <rootname> -seed <number> -workDir <dirname> \[-kick <number>\] \
   \[-measParamFile <filename>\] \[-bpmErrorFile\ <filename>]\
   \[-calculateBeamMoments <0|1> -rfBeamlineName <beamline-name>\] \
   \[-useKickFiles <0|1> -kickFileX <filename> -kickFileY <filename>\] \
   \[-correctTunes <0|1> -knobFile <filename>\] \[-correctOrbit <0|1>\] \[-removeAverageBpmError <0|1>\] \
   \[-verbose <0|1>\] \[-responseOnly <0|1>\] \[-calculateBeamMoments <0|1>\] "
if [info exists env(LOCO_BINDIR)] {
    set locoBinDir $env(LOCO_BINDIR)
} else {
    puts stderr "Error: LOCO_BINDIR environment variable is not defined"
    exit
}
if [info exists env(LOCO_TMPDIR)] {
    set tmpDir $env(LOCO_TMPDIR)
} else {
    set tmpDir $env(HOME)/tmp
}
if ![file exists $tmpDir] {exec mkdir -p $tmpDir}
source $locoBinDir/locoCommonProcedures
set tmpRoot $tmpDir/[APSTmpString]-genErr

set args $argv
set argCopy $args

set requiredVars [list lteFile beamline errorList elementListFile rootname seed workDir \
		      definitionFile machineName kick useKickFiles]
foreach var $requiredVars {set $var ""}
APSParseArguments $requiredVars

set nonrequiredVars [list paramFileList measParamFile removeAverageBpmError verbose responseOnly correctTunes \
			 calculateBeamMoments rfBeamlineName correctOrbit knobFile transform2measuredFormat \
			 outputTwissFile logFile kickFileX kickFileY bpmErrorFile elemByElemSR \
			 additionalErrorSources additionalErrorSourcesSeed averageOrbits treatNonBTSdiff nonBTSfile]
foreach var $nonrequiredVars {set $var ""}
foreach {correctTunes correctOrbit removeAverageBpmError verbose responseOnly transform2measuredFormat calculateBeamMoments elemByElemSR additionalErrorSourcesSeed averageOrbits treatNonBTSdiff} {0 0 1 0 0 0 0 0 1234567 0 0} {}
APSParseArguments $nonrequiredVars

set error 0
foreach var $requiredVars {
    if ![string length [set $var]] {set error 1; puts stderr "Error: $var is empty."}
}
if $calculateBeamMoments {
    if ![string length $rfBeamlineName] {set error 1; puts stderr "Error: rfBeamlineName is empty with calculateBeamMoments=1."}
}
if $error {
    puts stderr "Calling arguments: $argCopy"
    puts stderr $usage
    exit
}

#------ Make some files local (it is possible that individual nodes don't have access to remote file storage):
set copyfileList [list lteFile knobFile]
foreach filenameVar $copyfileList {
    if [string length [set $filenameVar]] {
	if {[string compare [set $filenameVar] $workDir/[file tail [set $filenameVar]]] != 0} {
	    #--- cp gives error if the target a broken link; "file exists" gives 0 for broken link, so just do "catch {delete}"
	    catch {file delete $workDir/[file tail [set $filenameVar]]}
	    exec cp -f [set $filenameVar] $workDir/[file tail [set $filenameVar]]
	    set $filenameVar $workDir/[file tail [set $filenameVar]]
	}
    }
}
if [string length $paramFileList] {
    set newParamFileList ""
    foreach filename $paramFileList {
	if {[string compare $filename $workDir/[file tail $filename]] != 0} {
	    file copy -force $filename $workDir/[file tail $filename]
	    lappend newParamFileList $workDir/[file tail $filename]
	} else {
	    lappend newParamFileList $filename
	}
    }
    set paramFileList $newParamFileList
}

#-------------------------------------------------------------------------------------------------------------------------

proc OutputMessage {args} {
    set logFile ""
    APSParseArguments {text verbose logFile}
    if $verbose {puts stdout $text}
    if [string length $logFile] {
        if [catch {open $logFile a} fid] { return -code error "$fid" }
	puts $fid $text
	close $fid
    }
}

#-------------------------------------------------------------------------------------------------------------------------

proc RemoveAllSpaces {line} {
    set strLength [string length $line]
    set line1 ""
    for {set I 0} {$I < $strLength} {incr I} {
	if {[string compare " " [string range $line $I $I]] != 0} {
	    append line1 [string range $line $I $I]
	}
    }
    return $line1
}

#-------------------------------------------------------------------------------------------------------------------------

proc CalculateMoments {args} {
    global tmpDir bRho deleteFiles
    APSParseArguments {lteFile paramFileList rfBeamlineName rootname}

    set tmpRoot $tmpDir/[APSTmpString]-mom
    set eleFile $tmpRoot.ele
    lappend deleteFiles $eleFile
    set energy [format %10.4e [expr $bRho * 299792458e-6]]
    if [catch {Fit_GenerateElegantFileFromLTE1 -eleOutputFile $eleFile \
		   -run_setup [list "lattice $lteFile use_beamline $rfBeamlineName rootname $rootname default_order 2 p_central_mev $energy"] \
		   -load_parameters [list "filename_list \"$paramFileList\""] \
		   -alter_elements [list "name * type KQUAD item N_KICKS value 50" "name * type KSEXT item N_KICKS value 3" \
					"name * type CSBEN* item N_KICKS value 100" "name * type CSBEN* item EDGE_ORDER value 1" \
					"name * type CSBEN* item INTEGRATION_ORDER value 4" \
					"name * type KQUAD item ISR value 1" "name * type CSBEN* item ISR value 1" \
					"name * type KQUAD item SYNCH_RAD value 1" "name * type CSBEN* item SYNCH_RAD value 1"] \
		   -closed_orbit [list "output $rootname.mom.orb"] \
		   -twissOutput [list "filename $rootname.mom.twi radiation_integrals 1"] \
		   -moments_output [list "filename $rootname.mom n_slices 100"] \
		   -run_control [list "n_steps 1"] \
		   -bunched_beam [list "n_particles_per_bunch 1"] \
	       } result] {
	return -code error "Fit_GenerateElegantFileFromLTE1: $result"
    }
    catch {file delete $eleFile.log}
    if [catch {exec elegant $eleFile > $eleFile.log} result] {
	return -code error "Error running elegant (see $eleFile.log): $result"
    }
    lappend deleteFiles $eleFile $rootname.mom.orb $rootname.mom.twi $eleFile.log
}

#-------------------------------------------------------------------------------------------------------------------------

proc ApplyCorrectorCalibrationCorrection {args } {
    global tmpDir deleteFiles
    APSParseArguments { hrmFile vrmFile hrmXFile vrmXFile hcorFile vcorFile}

    set tmpRoot $tmpDir/[APSTmpString]-calibCorr
    set hList [list $hrmFile $hrmXFile]
    set vList [list $vrmFile $vrmXFile]
    foreach fileList [list $hList $vList] corrFile [list $hcorFile $vcorFile] {
	if [exec sdds2stream $corrFile -rows=bare] {
	    foreach calcFile $fileList {
		set rows [lindex [exec sdds2stream $calcFile -rows=bare] 0]
		if [catch {exec sddsprocess $calcFile -pipe=out -reprint=col,BPMName,%s%s,BPMName,Plane \
			       | sddscombine -pipe -merge \
			       | sddstranspose -pipe -oldColumnNames=CorrectorNameValue -newColumnNames=BPMName \
			       | sddsxref -pipe $corrFile -fillIn -take=ParameterValue -match=CorrectorNameValue=ElementName \
			       | sddsprocess -pipe "-redef=col,%s,%s 1 ParameterValue + *,select=*,exclude=*Value" \
			       | sddsconvert -pipe -del=col,ParameterValue -del=para,* \
			       | sddstranspose -pipe -oldColumnNames=BPMName -newColumnNames=CorrectorNameValue \
			       | sddsprocess -pipe "-reedit=col,BPMName,e 1b 1d" \
			       | sddsbreak -pipe -rowlimit=$rows \
			       | sddsxref -pipe=in $calcFile $tmpRoot.out -leave=* -transfer=para,* } result] {
		    return -code error "Error applying calibration correction (calcFile: $calcFile; corrFile $corrFile): $result"
		}
		file copy -force $tmpRoot.out $calcFile
	    }
	}
    }
    lappend deleteFiles $tmpRoot.out
}

#-------------------------------------------------------------------------------------------------------------------------

proc RunOrbitCorrection {args} {

    global workDir deleteFiles
    APSParseArguments {outputFile fileRoot lteFile beamline paramFileList closedOrbit twissRefFile twissRefElement \
			   hcorListOC vcorListOC}

    set rootname $workDir/correctOrbit
    set eleFile $rootname.ele
    
    #------ Make use-correctors-for-steering file:
    exec sddsmakedataset -pipe=out -col=TagName,type=string -data=[join [concat $hcorListOC $vcorListOC] ,] \
	| sddsprocess -pipe=in $fileRoot.corrOC.param -def=col,ParameterValue,1 -print=col,ElementParameter,STEERING
    lappend deleteFiles $fileRoot.corrOC.param
    if [catch {Fit_MakeElementNameFromTagName -input $fileRoot.corrOC.param -addElementOccurence 1} result] {
	return -code error "Fit_MakeElementNameFromTagName: $result"
    }
    
    if $closedOrbit {set mode orbit; set matched 1} else {set mode trajectory; set matched 0}
    if [catch {Fit_GenerateElegantFileFromLTE1 -eleOutputFile $eleFile \
		   -run_setup [list "lattice $lteFile use_beamline $beamline rootname $rootname centroid %s.cen default_order 2"] \
		   -load_parameters [list "filename_list \"$paramFileList\""] \
		   -alter_elements [list "name * type MAXAMP item X_MAX value 0" "name * type MAXAMP item Y_MAX value 0" \
					"name * type *KICK* item STEERING value 0"] \
		   -load_parameters2 [list "filename $fileRoot.corrOC.param"] \
		   -correct [list "mode $mode trajectory_output %s.traj corrector_output %s.corr n_iterations 10 n_xy_cycles 3"] \
		   -closed_orbit [list "output %s.orb"] \
		   -twissOutput [list "matched $matched filename %s.twi reference_file $twissRefFile reference_element $twissRefElement radiation_integrals 1"] \
		   -run_control [list "n_steps 1"] \
		   -bunched_beam [list "n_particles_per_bunch 1"] \
	       } result] {
	return -code error "Fit_GenerateElegantFileFromLTE1: $result"
    }
    
    catch {file delete $eleFile.log}
    if [catch {exec elegant $eleFile > $eleFile.log} result] {
	return -code error "Error running elegant (see $eleFile.log): $result"
    }
    file copy -force $rootname.orb $fileRoot.orb
    lappend deleteFiles $eleFile.log $rootname.traj $eleFile $rootname.cen $rootname.corr $rootname.orb

    exec sddscombine $rootname.corr -pipe=out -merge \
	| sddsconvert -pipe -retain=col,ElementName,ParameterValue,ElementParameter,ElementOccurence \
	| sddsprocess -pipe=in $outputFile -print=col,ParameterMode,DIFFERENTIAL -print=col,VariableType,OrbitCorrection \
	-print=col,TagName,%s#%1d,ElementName,ElementOccurence
}

#-------------------------------------------------------------------------------------------------------------------------

proc RunTuneCorrection {args} {

    global workDir deleteFiles logFile
    APSParseArguments {outputFile fileRoot lteFile beamline paramFileList closedOrbit twissRefFile twissRefElement \
			   knobFile paramFileList0 verbose}

    if [string length $knobFile] {
	if ![file exists $knobFile] {return -code error "Knob file $knobFile does not exist."}
    } else {
	return -code error "Knob file variable is empty."
    }
    set rootname $workDir/correctTunes
    set eleFile $rootname.ele
    set twiFile $rootname.twi

    #------ Get initial tunes:
    if $closedOrbit {
	set twissList [list "matched 1 filename %s.twi"]
    } else {
	set twissList [list "matched 0 filename %s.twi reference_file $twissRefFile reference_element $twissRefElement"]
    }
    if [catch {Fit_GenerateElegantFileFromLTE1 -eleOutputFile $eleFile \
		   -run_setup [list "lattice $lteFile use_beamline $beamline rootname $rootname"] \
		   -load_parameters [list "filename_list \"$paramFileList0\""] \
		   -twiss_output $twissList \
		   -run_control [list "n_steps 1"] \
		   -bunched_beam [list "n_particles_per_bunch 1"] \
	       } result] {
	return -code error "Fit_GenerateElegantFileFromLTE1: $result"
    }
    if [catch {GetTunes -eleFile $eleFile -twiFile $twiFile} tuneList] {
	return -code error "GetTunes: $tuneList"
    }
    lappend deleteFiles $eleFile $twiFile $eleFile.log
    OutputMessage -logFile $logFile -verbose $verbose \
	-text "Model tunes are: Qx=[format %10.5f [lindex $tuneList 0]], Qy=[format %10.5f [lindex $tuneList 1]]"

    #------ Adjust tunes:
    if [catch {AdjustTunes \
		   -lteFile $lteFile \
		   -beamline $beamline \
		   -paramFileList $paramFileList \
		   -knobFile $knobFile \
		   -outputFile $outputFile \
		   -Qx [lindex $tuneList 0] \
		   -Qy [lindex $tuneList 1] \
		   -iterations 10 \
		   -tol 0.01 \
		   -closedOrbit $closedOrbit \
		   -twissRefFile $twissRefFile \
		   -twissRefElement $twissRefElement \
		   -verbose $verbose \
	       } result] {
        return -code error "AdjustTunes: $result"
    }

}
#----------------------------------------------------------------------------------------------------------------------------

proc AdjustTunes {args} {
    global workDir deleteFiles logFile
    APSParseArguments {lteFile beamline paramFileList outputFile Qx Qy knobFile iterations tol \
			   closedOrbit twissRefFile twissRefElement verbose}

    set rootname $workDir/adjustTunes
    set eleFile $rootname.ele
    set localParamFile $rootname.param
    set twiFile $rootname.twi
    lappend deleteFiles $eleFile $localParamFile $eleFile.log $twiFile

    if $closedOrbit {
	set twissList [list "matched 1 filename %s.twi"]
    } else {
	set twissList [list "matched 0 filename %s.twi reference_file $twissRefFile reference_element $twissRefElement"]
    }
    if [catch {Fit_GenerateElegantFileFromLTE1 -eleOutputFile $eleFile \
		   -run_setup [list "lattice $lteFile use_beamline $beamline rootname $rootname default_order 2"] \
		   -load_parameters [list "filename_list \"[concat $paramFileList $localParamFile]\""] \
		   -twiss_output $twissList \
		   -run_control [list "n_steps 1"] \
		   -bunched_beam [list "n_particles_per_bunch 1"] \
	       } result] {
	return -code error "Fit_GenerateElegantFileFromLTE1: $result"
    }

    #------ Getting initial tunes...
    if [catch {MakeParamFile -filename $localParamFile -knobFile $knobFile -xCoeff 0 -yCoeff 0} result] {
        return -code error "MakeParamFile: $result"
    }
    if [catch {GetTunes -eleFile $eleFile -twiFile $twiFile} tuneList] {
        return -code error "GetTunes: $tuneList"
    }
    file copy -force $localParamFile $localParamFile.0
    lappend deleteFiles $localParamFile $localParamFile.0
    set Qx0 [lindex $tuneList 0]
    set Qy0 [lindex $tuneList 1]
    OutputMessage -logFile $logFile -verbose $verbose \
	-text "Initial tunes are: Qx=[format %10.5f $Qx0], Qy=[format %10.5f $Qy0]"
    #------ Getting tune derivatives...
    if [catch {GetTuneDerivatives -eleFile $eleFile -knobFile $knobFile \
		   -paramFile $localParamFile -twiFile $twiFile -verbose $verbose} derivativeList] {
        return -code error "GetTuneDerivatives: $derivativeList"
    }
    set Qx1 [lindex $derivativeList 0]
    set Qy1 [lindex $derivativeList 1]
    set Qx2 [lindex $derivativeList 2]
    set Qy2 [lindex $derivativeList 3]
    #------ Return localParamFile to 0 after derivative calculation.
    if [catch {MakeParamFile -filename $localParamFile -knobFile $knobFile -xCoeff 0 -yCoeff 0} result] {
        return -code error "MakeParamFile: $result"
    }

    #------ Iterations
    OutputMessage -logFile $logFile -verbose $verbose -text "Adjusting tunes..."
    set done 0
    set I 0
    while {$done == 0} {
        set dQx [expr $Qx - $Qx0]
        set dQy [expr $Qy - $Qy0]
        set G1 [expr ($Qy2 * $dQx - $Qx2 * $dQy) / ($Qx1 * $Qy2 - $Qy1 * $Qx2)]
        set G2 [expr ($Qx1 * $dQy - $Qy1 * $dQx) / ($Qx1 * $Qy2 - $Qy1 * $Qx2)]
        if [catch {MakeParamFile -filename $localParamFile.iter -knobFile $knobFile -xCoeff $G1 -yCoeff $G2} result] {
            return -code error "MakeParamFile: $result"
        }
        if [catch {exec sddscombine $localParamFile $localParamFile.iter -pipe=out \
		       | sddsenvelope -pipe -sum=1,ParameterValue -copy=ElementName,ElementParameter,ParameterMode \
		       | sddsconvert -pipe=in $localParamFile.tmp -rename=col,ParameterValueSum=ParameterValue} result] {
	    return -code error "Error updating parameter file during iteration $I ($localParamFile $localParamFile.iter)"
	}
        file copy -force $localParamFile.tmp $localParamFile
        if [catch {GetTunes -eleFile $eleFile -twiFile $twiFile} tuneList] {
            return -code error "GetTunes: $tuneList"
        }
	OutputMessage -logFile $logFile -verbose $verbose -text "Iteration $I: $tuneList"
	set Qxprev $Qx0
	set Qyprev $Qy0
        set Qx0 [lindex $tuneList 0]
        set Qy0 [lindex $tuneList 1]
        if [catch {set dQx1 [expr abs($Qx0 - $Qxprev)]} result] {
	    OutputMessage -logFile $logFile -verbose $verbose \
		-text "Iteration $I, tune X: $result. Using nearest resonance tune."
	    set Qx0 [exec rpnl "$Qxprev 2 * 0.5 + int 2 /"]
	    set dQx1 [expr abs($Qx - $Qx0)]
	}
        if [catch {set dQy1 [expr abs($Qy0 - $Qyprev)]} result] {
	    OutputMessage -logFile $logFile -verbose $verbose \
		-text "Iteration $I, tune Y: $result. Using nearest resonance tune."
	    set Qy0 [exec rpnl "$Qyprev 2 * 0.5 + int 2 /"]
	    set dQy1 [expr abs($Qy - $Qy0)]
	}
	if {($dQx1 < $tol) && ($dQy1 < $tol)} {
	    set done 1
	} else {
	    incr I
	    if {$I > $iterations} {
		OutputMessage -logFile $logFile -verbose $verbose -text "Maximum number of iteration achieved."
		set done 1
	    }
	}
    }
    exec sddsprocess $localParamFile $outputFile -print=col,VariableType,TuneCorrection
#    file copy -force $localParamFile $outputFile
    lappend deleteFiles $localParamFile.tmp $localParamFile.iter

    OutputMessage -logFile $logFile -verbose $verbose \
	-text "Done. Final tunes are: Qx=[format %10.5f [lindex $tuneList 0]], Qy=[format %10.5f [lindex $tuneList 1]]"
}

#-------------------------------------------------------------------------------------------------------------------------

proc GetTuneDerivatives {args} {
    global logFile
    APSParseArguments {eleFile knobFile paramFile twiFile verbose}
    
    OutputMessage -logFile $logFile -verbose $verbose -text "Getting tune derivatives..."
    
    #--- List over non-perturbed tunes, x knob, and y knob:
    set coeff 0.001
    foreach coeffList [list "0 0" "$coeff 0" "0 $coeff"] listName [list tuneList0 tuneList1 tuneList2] {
        if [catch {MakeParamFile -filename $paramFile -knobFile $knobFile \
		       -xCoeff [lindex $coeffList 0] -yCoeff [lindex $coeffList 1]} result] {
            return -code error "MakeParamFile: $result"
        }
        if [catch {GetTunes -eleFile $eleFile -twiFile $twiFile} $listName] {
            return -code error "GetTunes: [set $listName]"
        }
    }
    
    set Qx1 [expr ([lindex $tuneList1 0] - [lindex $tuneList0 0]) / $coeff]
    set Qy1 [expr ([lindex $tuneList1 1] - [lindex $tuneList0 1]) / $coeff]
    set Qx2 [expr ([lindex $tuneList2 0] - [lindex $tuneList0 0]) / $coeff]
    set Qy2 [expr ([lindex $tuneList2 1] - [lindex $tuneList0 1]) / $coeff]
    return [list $Qx1 $Qy1 $Qx2 $Qy2]
}

#-------------------------------------------------------------------------------------------------------------------------

proc GetTunes {args} {
    APSParseArguments {eleFile twiFile}
    if [catch {exec elegant $eleFile > $eleFile.log} result] {
        return -code error "Error running elegant: $result"
    }
    set Qx [exec sdds2stream $twiFile -para=nux]
    set Qy [exec sdds2stream $twiFile -para=nuy]
    return [list $Qx $Qy]
}


#-------------------------------------------------------------------------------------------------------------------------

proc MakeParamFile {args} {
    APSParseArguments {filename knobFile xCoeff yCoeff}
    
    set fraction 0.5
    set tmpRoot /tmp/[APSTmpString]-tuneAdjust
    foreach pageName [list S:Tunex S:Tuney] coeff [list $xCoeff $yCoeff] tmpFile [list $tmpRoot.x $tmpRoot.y] {
	exec sddsprocess $knobFile $tmpFile -match=para,ControlName=$pageName \
	    "-reedit=col,ControlName,%/:CurrentAO//" \
	    "-redef=col,Weight,Weight $coeff * $fraction *"
    }
    exec sddscombine $tmpRoot.x $tmpRoot.y -pipe=out -merge \
	| sddssort -pipe -col=ControlName \
	| sddsbreak -pipe -change=ControlName \
	| sddsprocess -pipe -process=ControlName,first,ElementName -process=Weight,sum,ParameterValue \
	| sddscollapse -pipe \
	| sddsconvert -pipe -retain=col,ElementName,ParameterValue \
	| sddsprocess -pipe=in $filename -print=col,ElementParameter,K1 -print=col,ParameterMode,DIFFERENTIAL
    file delete $tmpRoot.x $tmpRoot.y
}

#-------------------------------------------------------------------------------------------------------------------------
proc ProtectDollarSigns {string} {
    set newString ""
    set stringLength [string length $string]
    for {set i 0} {$i < $stringLength} {incr i} {
	if {[string compare [string range $string $i $i] "$"] == 0} {
	    append newString "\\\$"
	} else {
	    append newString [string range $string $i $i]
	}
    }
    return $newString
}
#-------------------------------------------------------------------------------------------------------------------------


set deleteFiles ""
set seed0 $seed

set verbose1 $verbose
source $definitionFile
set verbose $verbose1

source $elementListFile

set bpmGainTiltString ""
if !$responseOnly {
    array set errorArray $errorList
    set errorVarList [list quadError quadTilt sextDispX sextDispY corCalib bpmGain corTilt bpmTilt bpmCoef bpmNoise bpmAver]
    foreach errorVar $errorVarList {
	set $errorVar $errorArray($errorVar)
    }

    #------ Generating files for errors:
    set errorNameList [list Quad Skew sextDispX sextDispY hCorr hCorrTilt vCorr vCorrTilt xBpm yBpm bpmTilt bpmCoef]
    array set elementListArr [list Quad $quadList Skew $skewList sextDispX $sextList sextDispY $sextList \
				  hCorr $hcorList hCorrTilt $hcorList vCorr $vcorList vCorrTilt $vcorList \
				  xBpm $xbpmList yBpm $ybpmList bpmTilt $bpmList bpmCoef $bpmList]
    array set errorArr [list Quad $quadError Skew $quadTilt sextDispX $sextDispX sextDispY $sextDispY \
			    hCorr $corCalib hCorrTilt $corTilt vCorr $corCalib vCorrTilt $corTilt \
			    xBpm $bpmGain yBpm $bpmGain bpmTilt $bpmTilt bpmCoef $bpmCoef]
    array set elementTypeArr [list Quad KQUAD Skew KQUAD sextDispX KSEXT sextDispY KSEXT hCorr HKICK hCorrTilt HKICK \
				  vCorr VKICK vCorrTilt VKICK xBpm MONI yBpm MONI bpmTilt MONI bpmCoef MONI]
    array set parameterArr [list Quad K1 Skew TILT sextDispX DX sextDispY DY hCorr CALIBRATION hCorrTilt TILT \
				vCorr CALIBRATION vCorrTilt TILT xBpm XCALIBRATION yBpm YCALIBRATION bpmTilt TILT bpmCoef COEF]
    array set parameterModeArr [list Quad DIFFERENTIAL Skew DIFFERENTIAL sextDispX DIFFERENTIAL sextDispY DIFFERENTIAL \
				    hCorr FRACTIONAL hCorrTilt DIFFERENTIAL vCorr FRACTIONAL vCorrTilt DIFFERENTIAL \
				    xBpm FRACTIONAL yBpm FRACTIONAL bpmTilt DIFFERENTIAL bpmCoef DIFFERENTIAL]
    set gLimit 2.0
    set fileListFinal ""
    foreach errorName $errorNameList {
	set elements $elementListArr($errorName)
	set filename $rootname.$errorName
	set error $errorArr($errorName)
	if {[llength $elements] && $error != 0} {}
	if {[llength $elements]} {
	    incr seed 5
	    exec sddsmakedataset -pipe=out -col=TagName,type=string -data=[join $elements ,] \
		| sddsprocess -pipe=in $filename "-rpnexpression=$seed srnd" "-def=col,ParameterValue,$gLimit grndl $error *" \
		"-edit=col,ElementName,TagName,%@#X@@ %@#Y@@ S?@#@ K" \
		"-scan=col,ElementOccurence,TagName,%ld,type=long,edit=%@#X@@ %@#Y@@ a z# d" \
		-print=col,ElementParameter,$parameterArr($errorName) -print=col,ParameterMode,$parameterModeArr($errorName) \
		-print=col,VariableType,$errorName -redef=para,Error,$error
	    lappend fileListFinal $filename
	    lappend deleteFiles $filename
	}
    }

    #--- For BTS error: if asked, change nonBTS errors according to an input file:
    if $treatNonBTSdiff {
	set nonBtsVarList [list nonBtsQuadError nonBtsSkewError nonBtsBpmError nonBtsBpmError nonBtsBpmTiltError]
	foreach errorName [list Quad Skew xBpm yBpm bpmTilt] nonBtsVar $nonBtsVarList {
	    if [catch {exec sddsprocess $nonBTSfile -pipe=out -match=col,VariableName=$nonBtsVar | sdds2stream -pipe=in -col=VariableValue} nonBtsError] {
		return -code error "Error reading nonBTSfile $nonBTSfile -- the file should have columns VariableName and VariableValue with the following \
                                    variables: [join $nonBtsVarList ,]: $nonBtsError"
	    }
	    set srDivider [expr $errorArr($errorName) / $nonBtsError]
	    exec sddsprocess $rootname.$errorName -nowarning \
		"-redef=col,ParameterValue,TagName \"BTS*\" strmatch ? ParameterValue : ParameterValue $srDivider / \$ "
	    file delete $rootname.${errorName}~
	}
    }

    #------ Combine sextupole displacement files:
    file delete $rootname.sextDisp
    if [llength $sextList] {
	set sextFileList ""
	if {$sextDispX != 0} {file copy -force $rootname.sextDispX $rootname.sextDisp}
	if {$sextDispY != 0} {
	    if {$sextDispX == 0} {
		file copy -force $rootname.sextDispY $rootname.sextDisp
	    } else {
		sddscombine $rootname.sextDispX $rootname.sextDispY $rootname.sextDisp -merge -overWrite
	    }
	}
	lappend deleteFiles $rootname.sextDisp
    }

    #------ Add Dispersion line to hCorr files:
    if {[llength $hcorList] && $corCalib != 0} {
	exec sddsmakedataset -pipe=out -col=ElementName,type=string -data=Dispersion \
	    | sddsprocess -pipe=in $tmpRoot.disp1 -def=col,ParameterValue,0.0 \
	    -print=col,ElementParameter,CALIBRATION -print=col,ParameterMode,DIFFERENTIAL -print=col,VariableType,hCorr
	exec sddscombine $rootname.hCorr $tmpRoot.disp1 $tmpRoot.disp2 -merge -overWrite
	file copy -force $tmpRoot.disp2 $rootname.hCorr
	exec sddsmakedataset -pipe=out -col=ElementName,type=string -data=Dispersion \
	    | sddsprocess -pipe=in $tmpRoot.disp3 -def=col,ParameterValue,0.0 \
	    -print=col,ElementParameter,TILT -print=col,ParameterMode,DIFFERENTIAL -print=col,VariableType,hCorrTilt
	exec sddscombine $rootname.hCorrTilt $tmpRoot.disp3 $tmpRoot.disp4 -merge -overWrite
	file copy -force $tmpRoot.disp4 $rootname.hCorrTilt
	lappend deleteFiles $tmpRoot.disp1 $tmpRoot.disp2 $tmpRoot.disp3 $tmpRoot.disp4
    }

    #------ Adding average BPM error:
    if {$bpmAver != 0} {
	if [llength $xbpmList] {
	    exec sddsprocess -nowarning $rootname.xBpm "-redef=col,ParameterValue,ParameterValue $bpmAver +"
	    lappend deleteFiles $rootname.xBpm~
	}
	if [llength $ybpmList] {
	    exec sddsprocess -nowarning $rootname.yBpm "-redef=col,ParameterValue,ParameterValue $bpmAver +"
	    lappend deleteFiles $rootname.yBpm~
	}
	
    }

    #------ Removing average bpm gain error:
    if $removeAverageBpmError {
	if [llength $xbpmList] {
	    exec sddsprocess -nowarning $rootname.xBpm -process=ParameterValue,average,Aver \
		"-redef=col,ParameterValue,ParameterValue Aver -"
	    lappend deleteFiles $rootname.xBpm~
	}
	if [llength $ybpmList] {
	    exec sddsprocess -nowarning $rootname.yBpm -process=ParameterValue,average,Aver \
		"-redef=col,ParameterValue,ParameterValue Aver -"
	    lappend deleteFiles $rootname.yBpm~
	}
    }
    eval exec sddscombine $fileListFinal $rootname.errors.param -merge -overWrite
    set finalErrorFileList $rootname.errors.param
    

    #------ Running elegant for orbit correction:
    if $correctOrbit {
	OutputMessage -logFile $logFile -verbose $verbose -text "Correcting orbit..."
	set localParamFileList $paramFileList
	foreach filename [list $rootname.Quad $rootname.Skew $rootname.sextDisp] {
	    if [file exists $filename] {set localParamFileList [concat $localParamFileList $filename]}
	}
	if [catch {RunOrbitCorrection -outputFile $rootname.orbCor \
		       -fileRoot $rootname -lteFile $lteFile -beamline $beamline -paramFileList $localParamFileList \
		       -closedOrbit $closedOrbit -twissRefFile $twissRefFile -twissRefElement $twissRefElement \
		       -hcorListOC $hcorListOC -vcorListOC $vcorListOC} result] {
	    return -code error "RunOrbitCorrection: $result"
	}
	set finalErrorFileList [concat $finalErrorFileList $rootname.orbCor]
	lappend deleteFiles $rootname.orbCor
    }

    #------ Correcting tunes:
    if $correctTunes {
	OutputMessage -logFile $logFile -verbose $verbose -text "Correcting tunes..."
	set localParamFileList $paramFileList
	foreach filename [list $rootname.Quad $rootname.Skew $rootname.sextDisp] {
	    if [file exists $filename] {set localParamFileList [concat $localParamFileList $filename]}
	}
	if [catch {RunTuneCorrection -outputFile $rootname.tuneCor \
		       -fileRoot $rootname -lteFile $lteFile -beamline $beamline -paramFileList $localParamFileList \
		       -closedOrbit $closedOrbit -twissRefFile $twissRefFile -twissRefElement $twissRefElement \
		       -knobFile $knobFile -paramFileList0 $paramFileList -verbose $verbose} result] {
	    return -code error "RunTuneCorrection: $result"
	}
	set finalErrorFileList [concat $finalErrorFileList $rootname.tuneCor]
	lappend deleteFiles $rootname.tuneCor
    }

    #------ Making final error file with all errors and orbit correction 
    #------ (NOTE: elements can appear several times in the final file, elegant reads parameter file row-by-row):
    set parDefString ""
    foreach errorType $errorVarList {
	append parDefString " -def=para,$errorType,[set $errorType] "
    }
    if [catch {eval exec sddscombine $finalErrorFileList -pipe=out -merge \
		   | sddsprocess -pipe=in $rootname.errors.param.tmp \
		   -print=para,Rootname,$rootname \
		   -def=para,Seed,$seed0,type=long \
		   -def=para,RemoveAverageBpmError,$removeAverageBpmError,type=long \
		   -def=para,CorrectTunes,$correctTunes,type=long \
		   -def=para,CorrectOrbit,$correctOrbit,type=long $parDefString} result] {
	return -code error "Error combining final parameter file (finalErrorFileList: $finalErrorFileList): $result"
    } else {
	file copy -force $rootname.errors.param.tmp $rootname.errors.param
	lappend deleteFiles $rootname.errors.param.tmp
    }

    set bpmGainTiltString "-bpmGainXFile $rootname.xBpm -bpmGainYFile $rootname.yBpm -bpmTiltFile $rootname.bpmTilt -bpmCoefFile $rootname.bpmCoef"

} else {
    #------ If responseOnly, need to create xbpm ybpm bpmTilt bpmCoef files for using inside ApplyBPMCorrections:
    if [string length $bpmErrorFile] {
	if [file exists $bpmErrorFile] {
	    foreach varType [list xBpm yBpm bpmTilt] colName [list GainX GainY Tilt] elemParam [list XCALIBRATION YCALIBRATION TILT] \
		paramMode [list FRACTIONAL FRACTIONAL DIFFERENTIAL] {
		    exec sddsconvert $bpmErrorFile -pipe=out -retain=col,TagName,ElementName,ElementOccurence,$colName \
			| sddsprocess -pipe -redef=col,ParameterValue,$colName -print=col,ElementParameter,$elemParam -print=col,ParameterMode,$paramMode \
			-print=col,VariableType,$varType \
			| sddsconvert -pipe=in $rootname.$varType -del=col,$colName
		}
	    exec sddsprocess $rootname.xBpm -nowarning "-reedit=col,TagName,S@#@i@#X@"
	    exec sddsprocess $rootname.yBpm -nowarning "-reedit=col,TagName,S@#@i@#Y@"
	    exec sddsprocess $rootname.bpmTilt $rootname.bpmCoef -redef=col,ParameterValue,0
	    set bpmGainTiltString "-bpmGainXFile $rootname.xBpm -bpmGainYFile $rootname.yBpm -bpmTiltFile $rootname.bpmTilt -bpmCoefFile $rootname.bpmCoef"
	    lappend deleteFiles $rootname.xBpm~ $rootname.yBpm~
	} else {
	    return -code error "File $bpmErrorFile does not exist."
	}
    }
    set hcorList [lreplace $hcorList [lsearch -exact $hcorList Dispersion] [lsearch -exact $hcorList Dispersion]]

    array set errorArray $errorList
    set bpmNoise $errorArray(bpmNoise)
}

#------ Calculating response matrix:

OutputMessage -logFile $logFile -verbose $verbose -text "Calculating response matrices..."

set rawLteFile $lteFile
set verbose1 $verbose
source $definitionFile
set verbose $verbose1

set beamlineName $beamline
#--- paramFileList: ideal lattice; measParamFile: file describing real lattice
if [string length $measParamFile] {
    set realLifeParamFileList "$paramFileList $measParamFile"
} else {
    set realLifeParamFileList $paramFileList
}
if !$responseOnly {set realLifeParamFileList [concat $realLifeParamFileList $rootname.errors.param]}

set hrmFile $rootname.hrm.calc
set vrmFile $rootname.vrm.calc
set hrmFileIdeal $rootname.hrm.ideal
set vrmFileIdeal $rootname.vrm.ideal
set hCorrList $hcorList
set vCorrList $vcorList
set kickX $kick
set kickY $kick
set coupledMatrix 2
set acceleratorCode elegant
set directCalc $directRMCalc
set fitDispersion 1

set useQsub $useQsubShort
set numberOfQsubTasks $numberOfQsubTasksShort
set waitTime $waitTimeShort
set waitInterval $waitIntervalShort
set qsubCommand $qsubCommandShort
set queueSystemName $queueSystemNameShort
set submissionPause $submissionPauseShort
set qsubRespProcCommand $qsubRespProcCommandShort
set bpmCorrectionStringList [list "" "$bpmGainTiltString -bpmNoise $bpmNoise -bpmNoiseSeed $seed"]
if $useKickFiles {
    set useKickFilesString " -useKickFiles $useKickFiles -kickFileX $kickFileX -kickFileY $kickFileY "
} else {set useKickFilesString ""}
if $useQsub {
    set qsubString " -numberOfQsubTasks $numberOfQsubTasks -waitTime $waitTime -waitInterval $waitInterval -qsubCommand \"[ProtectDollarSigns $qsubCommand]\" \
                     -queueSystemName $queueSystemName -submissionPause $submissionPause -qsubRespProcCommand \"[ProtectDollarSigns $qsubRespProcCommand]\" "
} else {set qsubString ""}
if !$closedOrbit {set twissRefString " -twissRefFile $twissRefFile -twissRefElement $twissRefElement "} else {set twissRefString ""}
if $directDispersionCalc {
    set directDispCalcString " -directDispersionCalc $directDispersionCalc -dispDirectParamList \"$dispDirectParamList\" "
} else {set directDispCalcString ""}
if {[string length $additionalErrorSources] != 0} {
    set additionalErrorSourceStringList [list "" " -additionalErrorSources \"$additionalErrorSources\" -additionalErrorSourcesSeed $additionalErrorSourcesSeed "]
} else {set additionalErrorSourceStringList [list "" ""]}
if $averageOrbits {set averageOrbitsStringList [list "" " -averageOrbits $averageOrbits "]} else {set averageOrbitsStringList [list "" ""]}
if {$verbose > 1} {set verboseString " -verbose $verbose "} else {set verboseString ""}
foreach localParamFileList [list $paramFileList $realLifeParamFileList] \
    localHrmFile [list $hrmFileIdeal $hrmFile] \
    localVrmFile [list $vrmFileIdeal $vrmFile] \
    localTwiFile [list $outputTwissFile.ideal $outputTwissFile] \
    outputString [list "Ideal:      " "With errors:"] \
    bpmCorrectionString $bpmCorrectionStringList \
    additionalErrorSourceString $additionalErrorSourceStringList \
    averageOrbitsString $averageOrbitsStringList {
	set command "$locoBinDir/locoCalculateResponseMatrix \
               -runMode getResponseMatrix \
	       -acceleratorCode $acceleratorCode \
	       -lteFile $lteFile \
	       -beamlineName $beamlineName \
	       -latticeParamFileList \"$localParamFileList\" \
	       -hrmFile $localHrmFile \
	       -vrmFile $localVrmFile \
	       -hCorrList \"$hCorrList\" \
	       -vCorrList \"$vCorrList\" \
	       -workDir $workDir \
	       -bRho $bRho \
	       -twiFile $localTwiFile \
               $twissRefString \
	       -dispFitWeight $dispFitWeight \
	       -directCalc $directCalc \
	       -kickX $kickX \
	       -kickY $kickY \
               $useKickFilesString \
	       -fixedOrbitLength $fixedOrbitLength \
	       -fitDispersion $fitDispersion \
	       -dispColumn $dispColumn \
	       -coupledMatrix $coupledMatrix \
	       -closedOrbit $closedOrbit \
	       -useDoubleOrbits $useDoubleOrbits \
	       -useQsub $useQsub \
               $qsubString \
               -elemByElemSR $elemByElemSR \
               $verboseString \
               $directDispCalcString \
               $additionalErrorSourceString \
               $averageOrbitsString \
               $bpmCorrectionString"
	OutputMessage -logFile $logFile -verbose $verbose -text "$command"
	if [catch {eval exec $command >&@ stdout} result] {
	    return -code error "locoCalculateResponseMatrix error: $result"
	}
	if $verbose {
	    set tuneList [exec sdds2stream $localTwiFile -para=nux,nuy]
	    puts stdout "$outputString Tunes: nux= [format %7.3f [lindex $tuneList 0]]; nuy= [format %7.3f [lindex $tuneList 1]]."
	}
	lappend deleteFiles $localHrmFile.XRM $localVrmFile.XRM
    }

#------ Beam moments calculation:
if $calculateBeamMoments {
    OutputMessage -logFile $logFile -verbose $verbose -text "Calculating beam moments..."
    if [catch {CalculateMoments -lteFile $lteFile -paramFileList $realLifeParamFileList -rootname $rootname \
		   -rfBeamlineName $rfBeamlineName} result] {
	return -code error "CalculateMoments: $result"
    }
}

OutputMessage -logFile $logFile -verbose $verbose -text "Postprocessing response matrix files..."

#------ Calculate beta function errors:
foreach {betaxRms betayRms etaxRms etayRms} \
    [exec sddschanges $outputTwissFile -pipe=out -base=$outputTwissFile.ideal -change=betax,betay,etax,etay -copy=betax,betay \
	 | sddsprocess -pipe "-redef=col,RelChangeBetax,ChangeInbetax betax /" "-redef=col,RelChangeBetay,ChangeInbetay betay /" \
	 -process=RelChangeBetax,rms,RelChangeBetaxRms -process=RelChangeBetay,rms,RelChangeBetayRms \
	 -process=ChangeInetax,rms,ChangeInetaxRms -process=ChangeInetay,rms,ChangeInetayRms \
	 | sdds2stream -pipe=in -para=RelChangeBetaxRms,RelChangeBetayRms,ChangeInetaxRms,ChangeInetayRms] {}
puts stdout "Rms errors: betax (rel): [format %6.3f $betaxRms]; betay (rel): [format %6.3f $betayRms]; etax (mm): [format %6.3f [expr $etaxRms * 1e3]]; etay (mm): [format %6.3f [expr $etayRms * 1e3]]."

#------ Calculate RM rms difference:
foreach ext [list hrm vrm] {
    exec sddsmatrixop $rootname.$ext.calc -pipe=out -push=$rootname.$ext.ideal -subt \
	| sddsxref -pipe=in $rootname.$ext.ideal $rootname.$ext.diff -take=TagName
    foreach filename [list $rootname.$ext.ideal $rootname.$ext.diff] varName [list rmsIdeal rmsDRM] {
	set $varName [exec sddsconvert $filename -pipe=out -del=col,$dispColumn \
			  | sddsmatrix2column -pipe \
			  | sddsprocess -pipe -process=Data,rms,Rms \
			  | sdds2stream -pipe=in -para=Rms]
    }
    if {[lindex $rmsIdeal 0] > 1e-5} {
	set ${ext}XRms [expr [lindex $rmsDRM 0] / [lindex $rmsIdeal 0]]
	set ${ext}Xstr "rel"
    } else {
	set ${ext}XRms [lindex $rmsDRM 0]
	set ${ext}Xstr "m/rad"
    }
    if {[lindex $rmsIdeal 1] > 1e-5} {
	set ${ext}YRms [expr [lindex $rmsDRM 1] / [lindex $rmsIdeal 1]]
	set ${ext}Ystr "rel"
    } else {
	set ${ext}YRms [lindex $rmsDRM 1]
	set ${ext}Ystr "m/rad"
    }
}
puts stdout "RM rms errors: HX ($hrmXstr): [format %6.3f $hrmXRms]; HY ($hrmYstr): [format %6.3f $hrmYRms]; VX ($vrmXstr): [format %6.3f $vrmXRms]; VY ($vrmYstr): [format %6.3f $vrmYRms]."

#------ Transforming into measured files format (splitting, transposing, and calibrating):
if $transform2measuredFormat {
    #------ Doing response files:
    exec sddsconvert $hrmFile $hrmFile.tmp -del=col,$dispColumn
    lappend deleteFiles $hrmFile.tmp
    set dirName [file dirname $rootname]
    set fileRoot [file tail $rootname]
    foreach \
	sourceFile [list $hrmFile.tmp $hrmFile.tmp $vrmFile $vrmFile] \
	rmFile [list $dirName/h$fileRoot.hrm $dirName/h$fileRoot.vrm $dirName/v$fileRoot.hrm $dirName/v$fileRoot.vrm] \
	keepPage [list 1 2 1 2] \
	kick [list $kickX $kickX $kickY $kickY] \
	calibFactor [list $hCalibFactor $hCalibFactor $vCalibFactor $vCalibFactor] {
	    set kick 1.0
	    exec sddsconvert $sourceFile -pipe=out -keepPage=$keepPage -del=col,BPMName \
		| sddsprocess -pipe "-redef=col,%s,%s $calibFactor * ,select=*,exclude=*Name" \
		-define=para,MeasuredFormat,$transform2measuredFormat,type=long \
		| sddstranspose -pipe=in $rmFile -newColumn=TagName -oldColumn=Corrector
	}
    
    #------ Doing dispersion file:
    exec sddsmakedataset $tmpRoot.dispCol -col=DispColumn,type=string -data=xDispersion,yDispersion
    lappend deleteFiles $tmpRoot.dispCol
    exec sddsconvert $hrmFile -pipe=out -retain=col,TagName,$dispColumn \
	| sddstranspose -pipe -newColumn=TagName -oldColumn=Dispersion \
	| sddscombine -pipe -merge \
	| sddsxref -pipe $tmpRoot.dispCol -take=DispColumn \
	| sddstranspose -pipe -newColumn=DispColumn -oldColumn=TagName \
	| sddsprocess -pipe "-redef=col,%s,%s $dispFitWeight / $dispCalibFactor /,select=*Dispersion" \
	| sddsconvert -pipe=in $rootname.slopes -del=col,Dispersion -rename=col,TagName=Rootname -del=para,*
} else {
    exec sddsprocess $hrmFile $hrmFile.mf -redefine=para,MeasuredFormat,$transform2measuredFormat,type=long
    exec sddsprocess $vrmFile $vrmFile.mf -redefine=para,MeasuredFormat,$transform2measuredFormat,type=long
    file copy -force $hrmFile.mf $hrmFile
    file copy -force $vrmFile.mf $vrmFile
    lappend deleteFiles $hrmFile.mf $vrmFile.mf
}

OutputMessage -logFile $logFile -verbose $verbose -text "Generating measurements: done."

eval file delete $deleteFiles

exit

