#!/bin/sh  
# \
  exec oagwish "$0" "$@"



set auto_path [linsert $auto_path 0  /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]
APSDebugPath

set IDScanArchivalDir /home/helios/oagData/sr/gapScans

proc MakeIDScanValuesFile {args} {
    set points 0
    set randomize 0
    set minimumGap 0
    set maximumGap 0
    set output ""
    APSStrictParseArguments {output points randomize minimumGap maximumGap}

    if ![string length $output] {
        return -code error "MakeIDScanValuesFile: no output file given"
    }
    if {$points<1 || $minimumGap>$maximumGap} {
        return -code error "MakeIDScanValuesFile: invalid grid parameters ($points, $minimumGap, $maximumGap)"
    }
    if $randomize {
        set randomizeCommand "sddsprocess -pipe -define=col,RN,rnd | sddssort -pipe -col=RN"
    } else {
        set randomizeCommand "cat"
    }

    if [catch {eval exec sddssequence -pipe=out -define=Index,type=long -sequence=begin=0,delta=1,number=$points \
                 | sddsprocess -pipe \
                 {"-define=column,GapSetpoint,Index 2 mod 0 == ? $minimumGap : $maximumGap \$ "} \
                 | $randomizeCommand > $output} result] {
        return -code error "MakeIDScanValuesFile: $result"
    }
}

proc RunGapScanExperiment {args} {
    set outputDir ""
    set outputRoot ""
    set runIndex -1
    set points 0
    set pause 0
    set randomize 0
    set minimumGap 0
    set maximumGap 0
    set IDSector 0
    set collectTunes 0
    set useVSA 0
    set statusCallback ""
    global IDScanArchivalDir
    APSStrictParseArguments {outputDir outputRoot runIndex points pause randomize \
                               minimumGap maximumGap IDSector statusCallback \
                               collectTunes useVSA}
    if $points<1 {
        if [string length $statusCallback] {
            eval $statusCallback {"points must be >1"}
        }
        return -code error "RunGapScanExperiment: points must be >1"
    }
    if $pause<0 {
        if [string length $statusCallback] {
            eval $statusCallback {"pause must be >=0"}
        }
        return -code error "RunGapScanExperiment: pause must be >=0"
    }
    if $IDSector<=0 {
        if [string length $statusCallback] {
            eval $statusCallback {"ID sector is invalid"}
        }
        return -code error "RunGapScanExperiment: ID sector is invalid"
    }
    set ID [format %02ld $IDSector]

    if ![string length $outputDir] {
        set outputDir .
    }
    if ![file exists $outputDir] {
        if {[catch {exec mkdir -p $outputDir} result] || \
              [catch {exec chmod a+rwx $outputDir} result]} {
            if [string length $statusCallback] {
                eval $statusCallback {"$result"}
            }
            return -code error "RunGapScanExperiment: $result"
        }
    }
    if ![string length $outputRoot] {
        if [string length $statusCallback] {
            eval $statusCallback {"output root is blank"}
        }
        return -code error "RunGapScanExperiment: output root is blank"
    }
    if {$runIndex<0} {
        set existingFiles [glob -nocomplain $outputDir/$outputRoot-\[0-9\]\[0-9\]*]
        set runIndex 0
        if [llength $existingFiles] {
            while 1 {
                set outputFile [format $outputDir/$outputRoot-%02ld $runIndex]
                if [lsearch -glob $existingFiles ${outputFile}*]==-1 break
                incr runIndex
            }
        }
    }
    set outputFile [format $outputDir/$outputRoot-%02ld $runIndex]
    if [string length $statusCallback] {
        eval $statusCallback {"Main output file is $outputFile"}
    }
 
    if [file exists $outputFile] {
        set response [APSMultipleChoice [APSUniqueName .] -question "$outputFile exists.  Overwrite it?" \
                        -labelList {Yes No} -returnList {Yes No} -name "File overwrite choice" ]
        switch $response {
            Yes { 
                if [catch {eval file delete [glob -nocomplain ${outputFile}*]} result] {
                    APSSetVarAndUpdate status "Unable to remove files!"
                    return
                }
                APSSetVarAndUpdate status "$outputFile will be overwritten."
                
            }
            No { 
                APSSetVarAndUpdate status "Give a new output file name."
                return -code error "RunGapScanExperiment: Duplicate filename"
            }
        }
    }

    if [catch {exec cavget -list=ID${ID}:DeviceLimit.DRVL -pend=5 } gapLimit] {
        if [string length $statusCallback] {
            eval $statusCallback {"$gapLimit"}
        }
        return -code error $gapLimit
    }
    if {$minimumGap<$gapLimit} {
        if [string length $statusCallback] {
            eval $statusCallback {"Minimum gap requested ($minimumGap) is less than limit of $gapLimit mm."}
            eval $statusCallback {"Will use limiting value." }
        }
        set minimumGap $gapLimit 
    }
    if {$maximumGap<$minimumGap} {
        if [string length $statusCallback] {
            eval $statusCallback {"Maximum gap less than minimum gap."}
        }
        return -code error "RunGapScanExperiment: Maximum gap less than minimum gap."
    }

    set valuesFile /tmp/[APSTmpString]
    APSAddToTempFileList $valuesFile
    if [catch {MakeIDScanValuesFile -output $valuesFile \
                 -minimumGap $minimumGap -maximumGap $maximumGap \
                 -points $points -randomize $randomize} result] {
        if [string length $statusCallback] {
            eval $statusCallback {"$result"}
        }
        return -code error $result
    }

    set template /tmp/[APSTmpString]
    set inputFile /tmp/[APSTmpString]
    APSAddToTempFileList $inputFile $template
    set scanFileRoot $IDScanArchivalDir/inputFiles/IDscan.template.exp.part
    if $collectTunes {
        if !$useVSA {
            lappend templateList ${scanFileRoot}1 ${scanFileRoot}2 ${scanFileRoot}3 
        } else {
            lappend templateList ${scanFileRoot}1 ${scanFileRoot}2VSA ${scanFileRoot}3 
        }
    } else {
        lappend templateList ${scanFileRoot}1 ${scanFileRoot}3 \
        }
    if [catch {eval exec cat $templateList > $template} result] {
        return -code error $result
    }
    if [catch \
          {exec replace $template $inputFile \
             -original=<sector>,<valuesFile>,<mainDir>,<runProc>,<rootname> \
             -replacement=$ID,$valuesFile,$IDScanArchivalDir/inputFiles,runProc,$outputFile} result] {
        if [string length $statusCallback] {
            eval $statusCallback {"$result"}
        }
        return -code error "RunGapScanExperiment: $result"
    }

    if [catch {exec cavput -list=ID${ID}:DeviceLimit=$gapLimit} result] {
        if [string length $statusCallback] {
            eval $statusCallback {"$result"}
        }
        return -code error $result
    }
    APSSRSetIOCAveraging -num2Ave 128 -enable 1 \
      -BPMList "A:P1 A:P2 A:P3 A:P4 B:P5 B:P4 B:P3 B:P2 B:P1" \
      -filterCoeff 0.1
    global runButtonWidget
    APSDisableButton $runButtonWidget
    catch {exec cavget -list=ID:AccessSecurity -num} accessSecurity
    catch {exec cavput "-list=ID:AccessSecurity=Machine Physics"}
    set execID [APSExecLog [APSUniqueName .] -width 120 -contextHelp \
                  "Execution window for sddsexperiment running ID$ID gap scan" \
                  -unixCommand "sddsexperiment -verbose $inputFile $outputFile -echo -summarize" \
                  -abortCallback "ScanDoneCallback -code abort -security $accessSecurity" \
                  -cancelCallback "ScanDoneCallback -code cancel -security $accessSecurity" \
                  -callback "ScanDoneCallback -code ok -fileRoot $outputFile -security $accessSecurity"]
    return -code ok "Scan started."
}

proc ScanDoneCallback {args} {
    set code ""
    set fileRoot ""
    set security 0
    APSStrictParseArguments {code fileRoot security} 

    global runButtonWidget
    APSEnableButton $runButtonWidget
    catch {exec cavput -list=ID:AccessSecurity=$security}
    bell
    switch $code {
        ok {
            set message "Scan completed successfully."
            APSSetVarAndUpdate status $message
            APSInfoWindow [APSUniqueName .] \
              -infoMessage $message
            bell
        }
        - {
            set message "Scan aborted or cancelled---check for proper condition of ID."
            APSSetVarAndUpdate status $message
            APSAlertBox [APSUniqueName .] \
              -errorMessage $message
            bell
            return
        }
    }
    if ![string length $fileRoot] {
        APSSetVarAndUpdate status "Scan aborted or cancelled---check for proper condition of ID."
        return
    }
    cd [file dirname $fileRoot]
    set VSAFileList [glob -nocomplain ${fileRoot}-vsa*]
    if [llength $VSAFileList] {
        if [catch {exec cavget -list=A014-IETS:BTC:SRSetFreqM -pendIoTime=25 -floatformat=%21.15e} rfFrequency] {
            return -code error "ScanDoneCallback: $rfFrequency"
        }
        set revFrequency [expr $rfFrequency / 1296.0]
        foreach VSAFile $VSAFileList {
            if [catch {exec sddsprocess $VSAFile -nowarning \
                         "-define=parameter,revFreq,$revFrequency,units=Hz" \
                         "-define=column,Tune,Frequency 1e6 * revFreq / = int - 0.5 > ? pop 1 swap - : pop \$ "} result] {
                return -code error "ScanDoneCallback: $result"
            }
            file delete -force ${VSAFile}~
        }
    }
    set files [glob -nocomplain ${fileRoot}*]
    if [llength $files] {
        APSSetVarAndUpdate status "Compressing [llength $files] files..."
        if [catch {eval exec gzip $files} result] {
            APSSetVarAndUpdate status "$result"
        }
        APSSetVarAndUpdate status "Done."
    } else {
        APSSetVarAndUpdate status "No data was collected!"
    }
}

proc MakeSectorsWidget {widget args} {
    global sector

    set parent ""
    APSParseArguments {parent}

    set w $parent$widget
    APSFrame $widget -parent $parent \
      -label "Insertion device selection" \
      -contextHelp {ID selection frame} 

    APSLabeledOutput .id -parent $parent$widget.frame \
      -label "ID: " -textVariable IDSector -width 4
    set sectorList [APSIDSectorList]
    for {set quad 1} {$quad<5} {incr quad} {
        set start [expr ($quad-1)*10+1]
        set end   [expr $start+9]
        set buttonList {}
        set valueList {}
        APSFrame .sector$quad -parent $w.frame 
        $w.frame.sector$quad.frame configure -relief flat
        for {set sector $start} {$sector<=$end} {incr sector} {
            set cbLabel $sector
            if {$sector<10} {set cbLabel "0$sector"}
            APSButton .sector$sector -parent $w.frame.sector$quad.frame \
              -text "$cbLabel" -size small \
              -command "SelectID $sector" \
              -contextHelp "Selects ID $sector for the experiment"
            if [lsearch $sectorList $sector]==-1 {
                APSDisableButton $w.frame.sector$quad.frame.sector$sector.button
            }
        }
    }
}

proc SelectID {sector} {
    global IDSector
    global nonArchivalWidgets archivalWidgets
    set IDSector $sector
    SwitchArchivalMode -archival 0 -nonArchivalWidgets $nonArchivalWidgets 
    SwitchArchivalMode -archival 1 -nonArchivalWidgets $nonArchivalWidgets 
}

proc SwitchArchivalMode {args} {
    set archival 0
    set nonArchivalWidgets ""
    APSStrictParseArguments {archival nonArchivalWidgets}
    global outputDir outputRoot runIndex 
    global IDScanArchivalDir IDSector 
    if $archival {
        foreach elem $nonArchivalWidgets {
            $elem configure -state disabled
        }
        set outputDir $IDScanArchivalDir/data/[exec date +%Y-%m%d]
        set outputRoot [format ID%02ld $IDSector]
        set runIndex -1
    } else {
        foreach elem $nonArchivalWidgets {
            $elem configure -state normal
        }
        set outputDir [pwd]
        set outputRoot ""
        set runIndex 0
    }
}

set nonArchivalWidgets ""
set archivalWidgets ""
proc MakeNonArchivalFrame {widget args} {
    set parent ""
    APSStrictParseArguments {parent}

    global nonArchivalWidgets archivalWidgets
    APSFrame $widget -parent $parent -label "Nonarchival output specification"
    set w $parent$widget.frame
    
    APSLabeledEntry .outputDir -parent $w -label "Directory: " \
      -width 60 -textVariable outputDir -contextHelp \
      "The directory for the files to which data will be saved."
    lappend nonArchivalWidgets $w.outputDir.entry
    APSLabeledEntry .output -parent $w    -label "Rootname:  " \
      -width 60 -textVariable outputRoot -contextHelp \
      "The root name for the files to which data will be saved.  The output file is <rootname>-<runIndex>.sdds"
    lappend nonArchivalWidgets $w.output.entry
    APSLabeledEntry .index -parent $w     -label "Run index: " \
      -width 60 -textVariable runIndex -contextHelp \
      "The run index for the next experiment.  Automatically incremented after each experiment."
    lappend nonArchivalWidgets $w.index.entry

    APSRadioButtonFrame .archival -parent $w -label "Archival: " \
      -variable archivalData -buttonList {Yes No} -valueList {1 0} \
      -orientation horizontal -commandList \
      {"SwitchArchivalMode -archival 1 -nonArchivalWidgets $nonArchivalWidgets" \
         "SwitchArchivalMode -archival 0 -nonArchivalWidgets $nonArchivalWidgets"} \
      -contextHelp "Selects whether to collect and review archival data.  \
      
 The data is placed in an archival area for long-term use." 

    SwitchArchivalMode -archival 1 -nonArchivalWidgets $nonArchivalWidgets 
}

proc MakeScanParametersFrame {widget args} {
    set parent ""
    APSStrictParseArguments {parent}

    APSFrame $widget -parent $parent -label "Scan parameters"
    set w $parent$widget.frame
    APSLabeledEntry .mingap -parent $w -label "Minimum gap (mm): " \
      -width 10 -textVariable minGap -gridPack "-sticky w -column 0 -row 0" -contextHelp \
      "The minimum gap value for the undulator scan. \
     Data will be collected only at the minimum and maximum gap points.  \
     If you enter -1, then the minimum allowed gap for the ID is used."
    APSLabeledEntry .maxgap -parent $w -label "Maximum gap (mm): " \
      -width 10 -textVariable maxGap -gridPack "-sticky w -column 0 -row 1" -contextHelp \
      "The maximum gap value for the undulator scan. \
     Data will be collected only at the minimum and maximum gap points."
    APSLabeledEntry .points -parent $w -label "Points: " \
      -width 10 -textVariable points -gridPack "-sticky w -column 0 -row 2" -contextHelp \
      "The number of points at which to take data.  Should be an even number so \
    that equal numbers of points are taken at the minimum and maximum gap."
    APSLabeledEntry .pause -parent $w -label "Pause (s): " \
      -width 10 -textVariable pause -gridPack "-sticky w -column 1 -row 0" -contextHelp \
      "The amount of time to pause after changing the gap and before taking data."
    APSRadioButtonFrame .tune -parent $w -label "Collect tunes: " \
      -orientation horizontal -gridPack "-sticky w -column 1 -row 1" \
      -variable collectTunes -buttonList {Yes No} -valueList {1 0} -contextHelp \
      "Selects whether to include tune data in the data collection.  Doing so may slow down the experiment significantly."
    APSRadioButtonFrame .vsa -parent $w -label "Use VSA: " \
      -orientation horizontal -gridPack "-sticky w -column 1 -row 2" \
      -variable useVSAForTunes -buttonList {Yes No} -valueList {1 0} -contextHelp \
      "If yes, then the SR HP VSA is used for tune data.  It is assumed to be set up ahead of time to look at the vertical tune only."
}

proc setStatusText {text} {
    global status
    set status $text
    update
}

proc ProcessAllGapScans {args} {
    global IDScanArchivalDir gapScanForceReprocessing 
    global gapScanStatusCallback gapScanDoPlots
    
    global gapScanPlotFileRoot gapScanPlotOutputTarget
    if [string compare $gapScanPlotOutputTarget file]==0 {
        if ![string length $gapScanPlotFileRoot] {
            return -code error "No output file supplied."
        }
    }

    set statusCallback APSNoOp
    set confirm 0
    APSStrictParseArguments {statusCallback confirm}
    if $confirm {
        if {![APSMultipleChoice [APSUniqueName .] \
              -question "Really? Process *all* scans?" \
              -labelList {Yes No} -returnList {1 0} \
                -name "AreYouSure"]} {
            return
        }
    }
    set gapScanForceReprocessing 1
    set gapScanStatusCallback $statusCallback
    set gapScanDoPlots 0
    for {set IDSector 1} {$IDSector<36} {incr IDSector} {
        eval $statusCallback {"Working on sector $IDSector"}
        set files [lsort [glob -nocomplain $IDScanArchivalDir/data/????-????/ID[format %02ld $IDSector]-??.gz]]
        if ![llength $files] {
            eval $statusCallback {"No files."}
            continue
        }
        regsub -all $IDScanArchivalDir/data/ $files {} tails0
        regsub -all .gz $tails0 {} tails
        if [catch {ProcessGapScan $tails} result] {
            eval $statusCallback {"$result"}
            return -code error "$result"
        }
    }
}

proc ProcessGapScanChoice {args} {
    set IDSector 0
    set force 0
    set statusCallback APSNoOp
    APSStrictParseArguments {IDSector force statusCallback} 
    global IDScanArchivalDir gapScanForceReprocessing gapScanStatusCallback
    global gapScanDoPlots
    set gapScanStatusCallback $statusCallback
    scan $IDSector %ld IDSector
    set gapScanForceReprocessing $force
    set gapScanDoPlots 1

    global gapScanPlotFileRoot gapScanPlotOutputTarget
    if [string compare $gapScanPlotOutputTarget file]==0 {
        if ![string length $gapScanPlotFileRoot] {
            eval $statusCallback {"No output file supplied."}
            return
        }
    }

    if $IDSector<=0 {
        eval $statusCallback {"Bad sector: $IDSector"}
        return
    }
    set files [lsort [glob -nocomplain $IDScanArchivalDir/data/????-????/ID[format %02ld $IDSector]-??.gz]]
    if [llength $files]==0 {
        eval $statusCallback {"No data for ID $IDSector"}
        return
    }
    regsub -all $IDScanArchivalDir/data/ $files {} tails0
    regsub -all .gz $tails0 {} tails
    set w [APSUniqueName .]
    APSScrolledListWindow $w -height 10 -name "ID $IDSector data choices" \
      -label "" -selectionVar dataToProcess -itemList $tails \
      -contextHelp \
      "Select the data you want to process or review.  The names are of the form <year>-<month><day>/ID<sector>." \
      -acceptButton 0 -clearButton 1 \
      -callback ProcessGapScan
#    $w.userFrame.sl.listbox configure -selectmode single
}

proc ProcessGapScan {dataRootList} {
    global IDScanArchivalDir gapScanForceReprocessing gapScanStatusCallback
    global gapScanDoPlots
    global gapScanStatusCallback gapScanDoPlots
    global gapScanUserTitle gapScanUserTopline 
    global gapScanPlotDevice gapScanPlotFileRoot gapScanPlotPrinter gapScanPlotOutputTarget

    if ![llength $dataRootList] return
    set statusCallback $gapScanStatusCallback

    switch $gapScanPlotOutputTarget {
        printer {
            if [string length $gapScanPlotPrinter] {
                set printCommand "| lpr -P$gapScanPlotPrinter"
                set deviceArg -device=$gapScanPlotDevice
            } else {
                set printCommand "| lpr"
                set deviceArg -device=$gapScanPlotDevice
            }
        } 
        file {
            set printCommand ""
            set deviceArg -device=$gapScanPlotDevice
        } 
        screen {
            set printCommand ""
            set deviceArg -device=motif
        }
    }

    foreach dataRoot $dataRootList {
        set year [string range $dataRoot 0 3]
        eval $statusCallback {"Processing/reviewing $dataRoot"}
        set fileRoot $IDScanArchivalDir/data/$dataRoot

        set primaryFile $fileRoot.gz
        set orbitChangesFile $fileRoot.ch
        set simOrbitFile $fileRoot.simorb
        set paramOrbitFile $fileRoot.param
        set simcorProcFile $fileRoot.simcor.proc

        if $gapScanForceReprocessing {
            foreach var {orbitChangesFile simOrbitFile paramOrbitFile simcorProcFile} {
                if {[file exists [subst \$$var]] && \
                      [catch {exec rm [subst \$$var]} result]} {
                    eval $statusCallback {"$result"}
                    return -code error $result
                }
            }
        }

        if {![file exists $orbitChangesFile] || \
              ![file exists $simOrbitFile] || \
              ![file exists $paramOrbitFile] || \
              ![file exists $simcorProcFile] } {
            scan [file tail $dataRoot] ID%02ld ID
            set ID [format %02ld $ID]
            set tmpRoot /tmp/[APSTmpString]

            if ![file exists $primaryFile] {
                eval $statusCallback {"$primaryFile not found!"}
                return -code error "$primaryFile not found!"
            }
            set tuneFiles [glob -nocomplain $fileRoot-tunes??.gz]
            APSAddToTempFileList $tmpRoot.tunes
            if [llength $tuneFiles]==0 {
                eval $statusCallback {"Warning: no NA/SA tune data found."}
            } else {
                eval $statusCallback {"[llength $tuneFiles] tune files found."}
                if [catch {eval exec sddscombine $tuneFiles -pipe=out \
                             | sddssmooth -pipe -passes=10 -points=3 -column=SR:TUNE* \
                             | sddsconvert -pipe -delete=parameter,yTune* \
                             | sddsprocess -pipe=in $tmpRoot.tunes \
                             -process=SR:TUNE1,max,yTuneVal \
                             -process=SR:TUNE1,max,yTune,functionOf=Tune,position} result] {
                    eval $statusCallback {"$result"}
                    return -code error "ProcessGapScan: $result"
                }
                if $gapScanDoPlots {
                    if [string compare $gapScanPlotOutputTarget file]==0 {
                        set printCommand "> $gapScanPlotFileRoot.tuneplot"
                    }
                    eval exec sddsplot $deviceArg \
                      -groupby=page -sep=page \
                      {-title=[APSMakeSafeQualifierString $gapScanUserTitle]} \
                      {-topline=[APSMakeSafeQualifierString $gapScanUserTopline]} \
                      -column=Tune,SR:TUNE1 $tmpRoot.tunes -split=page \
                      -parameter=yTune,yTuneVal $tmpRoot.tunes -graph=symbol,subtype=1 -split=page \
                      $printCommand &
                }
            }
            set HPVSAFileList [glob -nocomplain $fileRoot-vsa??.gz] 
            APSAddToTempFileList $tmpRoot.vsa
            if [llength $HPVSAFileList]==0 {
                eval $statusCallback {"Warning: no HP VSA tune data found."}
            } else {
                eval $statusCallback {"[llength $HPVSAFileList] HP VSA tune files found."}
                if [catch {eval exec sddscombine $HPVSAFileList -pipe=out \
                    | sddssmooth -pipe -passes=10 -points=3 -column=Waveform \
                    | sddsprocess -pipe=in $tmpRoot.vsa \
                    -process=Waveform,max,WaveformMax \
                    -process=Waveform,max,yTune,functionOf=Tune,position} result] {
                    eval $statusCallback {"$result"}
                    return -code error "ProcessGapScan: $result"
                }
                if $gapScanDoPlots {
                    if [string compare $gapScanPlotOutputTarget file]==0 {
                        set printCommand "> $gapScanPlotFileRoot.vsaplot"
                    }
                    eval exec sddsplot $deviceArg \
                      -groupby=page -sep=page \
                      {-title=[APSMakeSafeQualifierString $gapScanUserTitle]} \
                      {-topline=[APSMakeSafeQualifierString $gapScanUserTopline]} \
                      -column=Tune,Waveform $tmpRoot.vsa -split=page \
                      -parameter=yTune,WaveformMax $tmpRoot.vsa -ylabel= \
                      -graph=symbol,subtype=1 -split=page \
                      $printCommand &
                }
                    
            }
            set orbitData $tmpRoot.orbit
            APSAddToTempFileList $orbitData
            # delete sigma data
            # make file with columns for x and y orbit
            # import BPM data from xref file
            if [catch \
                  {exec sddsconvert $primaryFile -pipe=out -dele=col,Sigma* \
                     -retain=col,SigmaS35DCCT \
                     | sddscollect -pipe \
                     -collect=suffix=:msAve:x,column=x \
                     -collect=suffix=:msAve:y,column=y \
                     | sddsxref -pipe $IDScanArchivalDir/inputFiles/SR_msAve.mon \
                     -match=Rootname=BPMName -reuse=page -take=* \
                     | sddssort -pipe=in $orbitData  -column=Index} result] {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            # import tune data from tunes file, if there is one
            if {[file exists $tmpRoot.tunes] && \
                  [catch {exec sddsxref $orbitData $tmpRoot.tunes $orbitData.1 \
                            -leave=* -transfer=parameter,yTune 
                      exec mv $orbitData.1 $orbitData} result]} {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            if {[file exists $tmpRoot.vsa] && \
                  [catch {exec sddsxref $orbitData $tmpRoot.vsa $orbitData.1 \
                            -leave=* -transfer=parameter,yTune 
                      exec mv $orbitData.1 $orbitData} result]} {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            # find the values of the gap setpoints
            if [catch {exec sddsprocess -process=ID${ID}GapSetpoint,min,minGap \
                         -process=ID${ID}GapSetpoint,max,maxGap $primaryFile -pipe=out \
                         | sdds2stream -pipe -parameter=minGap,maxGap} gapList] {
                eval $statusCallback {"$gapList"}
                return -code error $result
            }
            set gap0 [lindex $gapList 0]
            set gap1 [lindex $gapList 1]

            # create a file for each state of the undulator
            # find the gap for each state and the envelope over
            # all orbits for each state
            foreach state {0 1} {
                set stateFile ${tmpRoot}.state$state
                APSAddToTempFileList $stateFile
                set gap [subst \$gap$state]
                if [catch \
                      {exec sddsprocess $orbitData -pipe=out \
                         -filter=param,S35DCCT,1,100 \
                         -filter=param,SigmaS35DCCT,0,1 \
                         "-test=parameter,$gap ID${ID}GapSetpoint - abs 1e-6 <" \
                         | tee $tmpRoot.tmp1 \
                         | sddsenvelope -pipe=in $stateFile \
                         -mean=x,y -copy=BPMName,Index } result] {
                    eval $statusCallback {"$result"}
                    return -code error $result
                }
                if [catch {exec sdds2stream -rows $tmpRoot.tmp1 | wc | token -n=1} result] {
                    eval $statusCallback {"$result"}
                    return -code error $result
                }
                eval $statusCallback {"$result orbits for state $state with gap $gap"}
                # process data to get mean tune for each state
                set result [list $state 1]
                if {[file exists $tmpRoot.tunes] && \
                      [catch {exec sddscollapse $tmpRoot.tmp1 -pipe=out \
                                | sddsprocess -pipe \
                                -filter=column,yTune,0,0.5 \
                                -process=yTune,average,yTuneMean \
                                -process=yTune,sigma,yTuneSigma \
                                | sdds2stream -pipe -param=yTuneMean,yTuneSigma} result]} {
                    eval $statusCallback {"$result"}
                    return -code error $result
                }
                if {[file exists $tmpRoot.vsa] && \
                      [catch {exec sddscollapse $tmpRoot.tmp1 -pipe=out \
                                | sddsprocess -pipe \
                                -filter=column,yTune,0,0.5 \
                                -process=yTune,average,yTuneMean \
                                -process=yTune,sigma,yTuneSigma \
                                | sdds2stream -pipe -param=yTuneMean,yTuneSigma} result]} {
                    eval $statusCallback {"$result"}
                    return -code error $result
                }
                set yTune$state [lindex $result 0]
                set yTuneSigma$state [lindex $result 1]
                exec rm $tmpRoot.tmp1
            }

            set deltaYTune [expr $yTune1-$yTune0]
            set deltaYTuneSigma [expr sqrt(pow($yTuneSigma1,2)+pow($yTuneSigma0,2))]

            # combine the data for the two states and find the changes
            # use statistics on changes to establish despiking parameters
            # despike the orbits to produce the smoothed delta orbit file
            APSAddToTempFileList $tmpRoot.stat
            if [catch \
                  {exec sddscombine ${tmpRoot}.state0 ${tmpRoot}.state1 -pipe=out \
                     | sddschanges -pipe \
                     -copy=Index,BPMName -changesIn=xMean,yMean \
                     | sddsprocess -pipe=in $tmpRoot.stat \
                     -process=*\[xy\]Mean,MAD,%sMAD} result] {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            catch {exec sdds2stream $tmpRoot.stat -param=ChangeInxMeanMAD} xMAD
            catch {exec sdds2stream $tmpRoot.stat -param=ChangeInyMeanMAD} yMAD

            if [catch \
                  {exec sddssmooth $tmpRoot.stat -pipe=out -new -column=ChangeInxMean \
                     -pass=0 -despike=neighbor=2,aver=2,pass=1,threshold=[expr 3*$xMAD] \
                     | sddssmooth -pipe -new -column=ChangeInyMean \
                     -pass=0 -despike=neighbor=6,aver=2,pass=1,threshold=[expr 2*$yMAD] \
                     | sddsprocess -pipe=in $orbitChangesFile \
                     -process=*xMeanSmoothed,stand,xStDev \
                     -process=*yMeanSmoothed,stand,yStDev \
                     "-print=parameter,Label2,x rms: %.3f mm  y rms: %.3f mm,xStDev,yStDev" \
                     "-print=parameter,Label1,Ref gap: $gap0 mm   Other gap: $gap1 mm"} result] {
                eval $statusCallback {"$result"}
                return -code error $result
            }
        }

        if $gapScanDoPlots {
            if [string compare $gapScanPlotOutputTarget file]==0 {
                set printCommand "> $gapScanPlotFileRoot.orbits"
            }
            set titleArg -title=@Label1
            set toplineArg "-topline=@Label2,edit=ei@  run: $dataRoot@"
            if [string length $gapScanUserTitle] {
                set titleArg "-title=[APSMakeSafeQualifierString $gapScanUserTitle]"
            }
            if [string length $gapScanUserTopline] {
                set toplineArg "-topline=[APSMakeSafeQualifierString $gapScanUserTopline]"
            }
            eval exec sddsplot $deviceArg $orbitChangesFile -groupby=names -sep=2 \
              -column=Index,ChangeIn?Mean*  -graph=line,vary \
              {$titleArg} {$toplineArg} $printCommand &
        }

        if {![file exists $simOrbitFile] || \
              ![file exists $paramOrbitFile] || \
              ![file exists $simcorProcFile] } {
            # convert delta orbit data to a form acceptable by elegant
            # remove A:P1's and B:P1's since position is often not known
            set xlimit 0.150
            set ylimit 0.150
            APSAddToTempFileList $tmpRoot.goodOrbit
            if [catch \
                  {exec sddsprocess $orbitChangesFile $tmpRoot.goodOrbit \
                     -match=col,BPMName=*A:P1,BPMName=*B:P1,\|,\! \
                     -print=column,ElementName,%s,BPMName \
                     -print=column,ParameterMode,absolute} result] {
                eval $statusCallback {"$result"}
                return -code $result
            }
            catch {exec chmod a+w $orbitChangesFile}

            # make lists of BPMs and orbit at places where orbit is good for
            #  x and then for y
            APSAddToTempFileList $tmpRoot.xOrbit $tmpRoot.yOrbit
            if {[catch \
                   {exec sddsprocess $tmpRoot.goodOrbit $tmpRoot.xOrbit \
                      -print=column,ElementParameter,DX \
                      -filter=col,ChangeInxMeanSmoothed,-$xlimit,$xlimit \
                      "-define=column,ParameterValue,ChangeInxMeanSmoothed 1e3 /" } result] || \
                  [catch \
                     {exec sddsprocess $tmpRoot.goodOrbit $tmpRoot.yOrbit \
                        -print=column,ElementParameter,DY \
                        -filter=col,ChangeInyMeanSmoothed,-$ylimit,$ylimit \
                        "-define=column,ParameterValue,ChangeInyMeanSmoothed 1e3 /" } result]} {
                eval $statusCallback {"$result"}
                return -code error $result
            }

            # make list of BPMs and orbit at places where x and y are good.
            APSAddToTempFileList $tmpRoot.usedOrbit
            if [catch \
                  {exec sddsselect $tmpRoot.xOrbit $tmpRoot.yOrbit -match=ElementName $tmpRoot.usedOrbit} result] {
                eval $statusCallback {"$result"}
                return -code error $result
            }

            # make list for all BPMs which have bad data, and assign a WEIGHT of
            #   zero so elegant won't use them
            APSAddToTempFileList $tmpRoot.badOrbit
            if [catch \
                  {exec sddsselect $orbitChangesFile $tmpRoot.usedOrbit \
                     -match=BPMName -invert -pipe=out -nowarning \
                     | sddsprocess -pipe=in $tmpRoot.badOrbit \
                     -print=column,ElementName,%s,BPMName \
                     -print=column,ParameterMode,absolute \
                     -print=column,ElementParameter,WEIGHT \
                     -define=column,ParameterValue,0} result] {
                eval $statusCallback {"$result"}
                return -code error $result
            }

            # Decide which lattice to use.  Use 19.3 lattice after Jan 1 1999.
            if $year>=1999 {
                set lteFile aps-19.3-N.lte
                set twissFile aps-19.3.twi
            } else {
                set lteFile aps-14.3-N.lte
                set twissFile aps-14.3.twi
            }
            # combine the lists to make the final parameters file for elegant
            APSAddToTempFileList $tmpRoot.param
            if {[catch \
                   {exec sddscombine $tmpRoot.xOrbit $tmpRoot.yOrbit $tmpRoot.badOrbit -pipe=out -merge \
                      | sddsxref -pipe $IDScanArchivalDir/inputFiles/$twissFile \
                        -match=ElementName -take=s -reuse=rows \
                      | sddssort -col=s,incr -pipe \
                      | sddsconvert -pipe=in $tmpRoot.param -dele=col,:* -dele=param,*} result] || \
                  [catch {exec cp $tmpRoot.param $paramOrbitFile} result] } {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            catch {exec chmod a+w $paramOrbitFile}

            # prepare and run simulation of the delta orbit
            scan $ID %ld id
            APSAddToTempFileList $tmpRoot.lte $tmpRoot.ele $tmpRoot.simorb $tmpRoot.simcor $tmpRoot.log
            
            if {[catch {exec replace $IDScanArchivalDir/inputFiles/$lteFile \
                          $tmpRoot.lte "-orig=<N>" -repl=[expr $id+1]} result] || \
                  [catch {exec replace $IDScanArchivalDir/inputFiles/apsN.ele \
                            $tmpRoot.ele "-orig=<rootname>" -repl=$tmpRoot} result]} {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            
            eval $statusCallback {"Running 'elegant' simulation."}
            if {[catch {exec elegant $tmpRoot.ele >& $tmpRoot.log} result] || \
                  [catch {exec cp $tmpRoot.simorb $simOrbitFile} result]} {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            
            if {[catch {exec sddsprocess $simOrbitFile \
                          -match=parameter,Stage=corrected $simOrbitFile.1} result] || \
                  [catch {exec mv $simOrbitFile.1 $simOrbitFile} result]} {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            catch {exec chmod a+w $simOrbitFile}

            set minGap [expr $gap0<$gap1 ? $gap0 : $gap1 ]
            set maxGap [expr $gap0>$gap1 ? $gap0 : $gap1 ]
            APSAddToTempFileList $simcorProcFile.1
            if [catch \
                  {exec sddsprocess -nowarning $tmpRoot.simcor -pipe=out \
                     -process=ParameterValue,sum,IntegratedKick \
                     -process=ParameterValue,first,FirstKick \
                     -process=ParameterValue,last,SecondKick \
                     -define=parameter,Rigidity,23.5e6,units=G-cm \
                     -define=parameter,UndLength,240,units=cm \
                     "-define=parameter,FirstIntegral,IntegratedKick Rigidity *,units=G-cm" \
                     "-define=parameter,SecondIntegral,FirstKick Rigidity * UndLength *,units=G-cm\$a2\$n" \
                     "-print=param,Rootname,$dataRoot" \
                     "-define=parameter,MinimumGap,$minGap,units=mm" \
                     "-define=parameter,MaximumGap,$maxGap,units=mm" \
                     "-define=parameter,deltaYTuneMean,$deltaYTune" \
                     "-define=parameter,deltaYTuneSigma,$deltaYTuneSigma" \
                     | sddsxref -pipe $orbitData -leave=* -transfer=parameter,Time \
                     | sddstimeconvert -pipe=in -breakdown=parameter,Time,text=TimeStamp \
                     $simcorProcFile.1} result] {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            set tmpFile /tmp/[APSTmpString]
            APSAddToTempFileList $tmpFile.x $tmpFile.y $tmpFile
            foreach coord {x y} {
                if [catch {exec sddsconvert $orbitChangesFile $tmpFile.$coord \
                             -delete=column,* \
                             -retain=parameter,ChangeIn${coord}*MAD,${coord}StDev \
                             -edit=parameter,*,%/ChangeIn$coord/ChangeInOrbit/%/${coord}StDev/DeltaOrbitStDev/ \
                         } result] {
                    eval $statusCallback {"$result"}
                    return -code error $result
                }
            }
            if [catch {exec sddscombine $tmpFile.x $tmpFile.y $tmpFile
                exec sddsxref $simcorProcFile.1 $tmpFile \
                         $simcorProcFile -leave=* -transfer=param,*} result] {
                eval $statusCallback {"$result"}
                return -code error $result
            }
            catch {exec chmod a+w $simcorProcFile}
        }

        if [catch {sdds open $simcorProcFile r} fid] {
            eval $statusCallback {"$fid"}
            return -code error $fid
        }
        set minGap [format %.2f [APSGetSDDSParameter -sddsFD $fid \
                                   -parameter MinimumGap -page 1]]
        set maxGap [format %.2f [APSGetSDDSParameter -sddsFD $fid \
                                   -parameter MaximumGap -page 1]]
        set Integ1x [format %.1f [APSGetSDDSParameter -sddsFD $fid \
                                    -parameter FirstIntegral -page 1]]
        set Integ2x [format %.1f [APSGetSDDSParameter -sddsFD $fid \
                                    -parameter SecondIntegral -page 1]]
        set Integ1y [format %.1f [APSGetSDDSParameter -sddsFD $fid \
                                    -parameter FirstIntegral -page 2]]
        set Integ2y [format %.1f [APSGetSDDSParameter -sddsFD $fid \
                                    -parameter SecondIntegral -page 2]]
        catch {sdds close $fid}

        if $gapScanDoPlots {
            set command ""
            if [string length $gapScanUserTitle] {
                set title $gapScanUserTitle
            } else {
                set title "Measured and fit orbit change for $dataRoot"
            }
            if [string length $gapScanUserTopline] {
                set topline1 $gapScanUserTopline
                set topline2 $gapScanUserTopline
            } else {
                set topline1 "\$gD\$rgap: \[$minGap\\, $maxGap\] mm   \$gD\$r\$sI\$eBdl\\=$Integ1x G-cm  \$gD\$r\$sII\$eBdl\\=$Integ2x G-cm\$a2\$n"
                set topline2 "\$gD\$rgap: \[$minGap\\, $maxGap\] mm   \$gD\$r\$sI\$eBdl\\=$Integ1y G-cm  \$gD\$r\$sII\$eBdl\\=$Integ2y G-cm\$a2\$n"
            }
            lappend command -labelsize=0.027 -toptitle \
              -factor=yMultiplier=1e6 \
              "-title=[APSMakeSafeQualifierString $title]" \
              -column=s,ParameterValue -graph=dot,type=1,subtype=10 -match=col,ElementParameter=DX $paramOrbitFile \
              "-ylabel=horizontal orbit change (\$gm\$rm)" \
              "-topline=[APSMakeSafeQualifierString $topline1]" \
              -column=s,x $simOrbitFile \
              -column=s,x $simOrbitFile -match=col,ElementName=UNDCENT -graph=sym,type=2,subtype=2,scale=2 -end \
              -column=s,ParameterValue -graph=dot,type=1,subtype=10 -match=col,ElementParameter=DY $paramOrbitFile \
              "-topline=[APSMakeSafeQualifierString $topline2]" \
              "-ylabel=vertical orbit change (\$gm\$rm)" \
              -column=s,y $simOrbitFile \
              -column=s,y $simOrbitFile -match=col,ElementName=UNDCENT -graph=sym,type=2,subtype=2,scale=2 
            if [string compare $gapScanPlotOutputTarget file]==0 {
                set printCommand "> $gapScanPlotFileRoot.tuneplot"
            }
            if [string compare $gapScanPlotOutputTarget file]==0 {
                set printCommand "> $gapScanPlotFileRoot.fits"
            }

            eval exec sddsplot $deviceArg $command $printCommand &
            eval $statusCallback {"Plot launched."}
        }
    }
    after 2000 
    # APSDeleteTempFiles
}

proc ShowMeasurementHistory {args} {
    set IDSector 0
    APSStrictParseArguments {IDSector} 
    global IDScanArchivalDir
    scan $IDSector %ld IDSector
    if $IDSector<=0 {
        APSSetVarAndUpdate status "Bad sector: $IDSector"
        return
    }
    set files \
      [lsort [glob -nocomplain \
                $IDScanArchivalDir/data/????-????/ID[format %02ld $IDSector]-??.simcor.proc]]
    if [llength $files]==0 {
        APSSetVarAndUpdate status "No data for ID $IDSector"
        return
    }
    
    set tmpFile /tmp/[APSTmpString]
    if [catch {eval exec sddscombine $files -pipe=out -collapse \
                 | sddssort -pipe -column=Plane -column=Time \
                 | sddsprocess -pipe -edit=column,Plane0,Plane,4fD \
                 -reedit=column,TimeStamp,S/./2D \
                 -reedit=column,Rootname,2D/ \
                 | sddsprintout -pipe=in $tmpFile -width=130 \
                 {"-title=Gap scan history for ID $IDSector"} \
                 -column=TimeStamp,format=%19s \
                 -column=Rootname,format=%6s \
                 -column=Plane0,format=%5s,label=Plane \
                 -column=MinimumGap,label=MinGap,format=%6.2f \
                 -column=MaximumGap,label=MaxGap,format=%6.2f \
                 -column=DeltaOrbitStDev,format=%10.4f,label=DeltaOrbStDev \
                 -column=ChangeInOrbitMeanMAD,format=%10.4f,label=DeltaOrbMAD \
                 -column=FirstIntegral,format=%7.1f,label=Integ1 \
                 -column=SecondIntegral,format=%7.1f,label=Integ2 \
                 -column=deltaYTuneMean,format=%7.2g,label=dYTune \
                 -column=deltaYTuneSigma,format=%7.2g,label=dYTuneSig } result] {
        APSSetVarAndUpdate status "$result"
        return
    }
    APSFileDisplayWindow [APSUniqueName .] -fileName $tmpFile \
      -comment "Measurement History for ID[format %02ld $IDSector]" \
      -width 130 -deleteOnClose 1 -printCommand "enscript -r"

}

proc MakePlotControlFrame {widget args} {
    set parent ""
    APSStrictParseArguments {parent}

    global gapScanUserTitle gapScanUserTopline 
    global gapScanPlotDevice gapScanPlotFileRoot gapScanPlotPrinter gapScanPlotOutputTarget
    set gapScanUserTitle ""
    set gapScanUserTopline ""
    set gapScanPlotDevice post
    set gapScanPlotOutputTarget screen
    set gapScanPlotFileRoot ""
    global env
    if [info exists env(PlotPrinter)] {
        set gapScanPlotPrinter $env(PRINTER)
    } else {
        set gapScanPlotPrinter mcr1
    }


    APSFrame $widget -parent $parent -label "Plot Controls"
    set w $parent$widget.frame

    APSLabeledEntry .title -parent $w -label "Title: " -width 60 -textVariable \
      gapScanUserTitle -contextHelp "User-supplied title for the plot.  If blank, one is generated for you."
    APSLabeledEntry .topline -parent $w -label "Topline: " -width 60 -textVariable \
      gapScanUserTopline -contextHelp \
      "User-supplied topline for the plot.  If blank, one is generated for you."
    APSRadioButtonFrame .outputTo -parent $w -label "Output to: " \
      -variable gapScanPlotOutputTarget -buttonList {File Printer Screen} \
      -valueList {file printer screen} -orientation horizontal -contextHelp \
      "Choose whether to send graphics output to a file, a printer, or the screen."
    APSRadioButtonFrame .plotDevice -parent $w -label "File/printer device: " \
      -variable gapScanPlotDevice \
      -buttonList {PS CPS EPS CEPS} \
      -valueList {post cpost eps ceps} \
      -orientation horizontal -contextHelp \
      "Specify the type of graphics output for printer or file.  PS is B&W postscript, CPS is color postscript, EPS is B&W encapsulated postscript, and CEPS is color encapsulated postscript."
    APSLabeledEntry .plotFileRoot -parent $w -label "Plot output file rootname: " -width 60 \
      -textVariable gapScanPlotFileRoot -contextHelp \
      "Give a rootname for files for graphics output.  Relevant only if you choose to output to a file above."
    APSLabeledEntry .printer -parent $w -label "Printer: " -width 60 \
      -textVariable gapScanPlotPrinter -contextHelp \
      "Give the name of a printer for graphics output.  Relevant only if you choose to output to a printer above."
    
}

proc MakeActionWidget {widget args} {
    set parent ""
    set expert 0

    APSStrictParseArguments {parent expert}
    APSFrame $widget -parent $parent -label ""
    set w $parent$widget.frame
    global runButtonWidget
    set runButtonWidget $w.run.button

    APSButton .run -parent $w -text Run -command \
      {catch {RunGapScanExperiment -IDSector $IDSector -outputRoot $outputRoot -runIndex $runIndex \
                -outputDir $outputDir \
                -points $points -pause $pause \
                -randomize $randomize -minimumGap $minGap -maximumGap $maxGap \
                -collectTunes $collectTunes \
                -useVSA $useVSAForTunes \
                -statusCallback setStatusText} status; update} \
      -contextHelp "Does a gap scan."

    APSButton .review -parent $w -text Process/Review \
      -command {ProcessGapScanChoice -IDSector $IDSector \
            -statusCallback "APSSetVarAndUpdate status"} -contextHelp \
      "Allows processing and review of data."

    APSButton .freview -parent $w -text "Forced Process/Review" \
      -command {ProcessGapScanChoice -IDSector $IDSector -force 1 -statusCallback "APSSetVarAndUpdate status"} -contextHelp \
      "Allows processing and review of data, but forces reanalysis of the data in spite of any indications that analysis is already available."

    if $expert {
        APSButton .processAll -parent $w -text "Process All" \
        -command {ProcessAllGapScans -statusCallback "APSSetVarAndUpdate status" -confirm 1} \
        -contextHelp "Processes or reprocesses *all* gap scans in the archive.  Used when the processing algorithm is changed."
    }

    APSButton .trend -parent $w -text "Show History" \
      -command {ShowMeasurementHistory -IDSector $IDSector} -contextHelp \
      "Shows the history of measurement results for the chosen ID."
}

set CVSRevisionAuthor "\$Revision: 1.35 $ \$Author: soliday $"
APSApplication . -name UndulatorGapScan -version $CVSRevisionAuthor \
  -overview "Does undulator gap scans to determine how much an undulator affects the orbit and tunes."

set args $argv
set expert 0
APSStrictParseArguments {expert}

set outputDir [pwd]
set outputRoot ""
set runIndex 0
set points 10
set pause 2
set IDSector 1
set minGap -1
set maxGap 45
set points 20
set randomize 0
set collectTunes 1
set useVSAForTunes 0
set archivalData 1

set status Ready.
APSScrolledStatus .status -parent .userFrame -textVariable status -width 60 -height 4

MakeSectorsWidget .sectors -parent .userFrame
MakeNonArchivalFrame .nonarchival -parent .userFrame 
MakeScanParametersFrame .scan -parent .userFrame
MakePlotControlFrame .pcontrol -parent .userFrame
MakeActionWidget .action -parent .userFrame -expert $expert


