#!/bin/sh
# \
exec oagwish "$0" "$@"



set auto_path [linsert $auto_path 0  /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0  /usr/local/oag/lib_patch/$env(HOST_ARCH)]
APSDebugPath
set CVSRevisionAuthor "\$Author: borland $"

# set FilterWheelDataFile /home/helios/oagData/linac/FELData/NDFilters.sdds

set VLDGroupList {9 8 7 6 5 4 3 2 1 0 9a}

proc setStatus {text} {
    APSSetVarAndUpdate status "[clock format [clock seconds] -format %H:%M:%S]  $text"
}

proc MakeVLDWidgets {widget args} {
    set parent ""
    set group 0
    APSStrictParseArguments {group parent}

    APSFrame $widget -parent $parent -label "Group $group" -noPack 1
    set w $parent$widget.frame

    global numberOfFiles listOfRootnames NDFilter ROI spotROISize sizeLines
    global NDFilterWheel
    global ROI blankOut backgroundHalfWidth symmetricBackgroundRemoval lonerRemoval 
    global saturationLevel centroidTolerance allowedSaturatedPixels filterWheelFile
    ResetVLDConfig
    # set backgroundHalfWidth($group) 2
    # set lonerRemoval($group) 0
    # set symmetricBackgroundRemoval($group) 0
    # set numberOfFiles($group) 0
    # set listOfRootnames($group) ""
    # set NDFilter($group) 0
    # set ROI(x0,$group) 50
    # set ROI(x1,$group) 718
    # set ROI(y0,$group) 0
    # set ROI(y1,$group) 479
    # set NDFilterWheel(1,$group) -1
    # set NDFilterWheel(2,$group) -1
    # set NDFilterWheel(3,$group) -1
    # set spotROISize(x,$group) 150
    # set spotROISize(y,$group) 150
    # set sizeLines(x,$group) 3
    # set sizeLines(y,$group) 3
    # set blankOut(x0,$group) -1
    # set blankOut(x1,$group) -1
    # set blankOut(y0,$group) -1
    # set blankOut(y1,$group) -1
    # set saturationLevel($group) 254
    # set centroidTolerance($group) 10
    # set allowedSaturatedPixels($group) 4
    # set filterWheelFile($group) /home/helios/oagData/linac/FELData/NDFilters530nm.sdds

    APSLabeledEntry .filterwheelfile -parent $w -label "Filter wheel file: " \
        -width 80 -textVariable filterWheelFile($group)
    APSFrameGrid .fg -parent $w -xList {x1 x2} -yList {y1 y1a y2 y3 y4 y5 y6 y7} 
    APSLabeledEntryFrame .ndfilters -parent $w.fg.x1.y1 -label "Filter wheels*: " \
      -variableList "NDFilterWheel(1,$group) NDFilterWheel(2,$group) NDFilterWheel(3,$group)" \
      -orientation horizontal -width 8 \
      -contextHelp "Enter the filter wheel settings for this group.  If all are -1, then the user-supplied filter value is used.  Applied at collation stage."
    APSButton .ndfiltercalc -parent $w.fg.x2.y1 -text "Filter wheels->ND filter level" \
       -command "ConvertFilterWheelToFilterLevel -group $group"
    APSLabeledEntry .ndfilter -parent $w.fg.x1.y1a -label "ND filter level*: " \
      -textVariable NDFilter($group) \
      -contextHelp "Enter the neutral density filter value for this group.  Applied at collation stage."

    APSLabeledEntry .sensitivity -parent $w.fg.x2.y1a -label "Camera sensitivity: " \
      -textVariable cameraSensitivity($group) \
      -contextHelp "Shows and allows modification of camera sensitivity."

    APSLabeledEntryFrame .roi -parent $w.fg.x1.y2 -label "ROI (xLo, xHi, yLo, yHi): " -orientation horizontal \
      -variableList "ROI(x0,$group) ROI(x1,$group) ROI(y0,$group) ROI(y1,$group)" \
      -width 4 -contextHelp "Enter the ROI in pixel units, in the order xLow, xHigh, yLow, yHigh."
    APSLabeledEntryFrame .boroi -parent $w.fg.x2.y2 \
      -label "BOR (xLo, xHi, yLo, yHi): " -orientation horizontal \
      -variableList "blankOut(x0,$group) blankOut(x1,$group) blankOut(y0,$group) blankOut(y1,$group)" \
      -width 3 -contextHelp "Enter the boundaries of the optional blank-out region in pixel units, in the order xLow, xHigh, yLow, yHigh.  Any pixels inside the BOR are set to zero."
    APSLabeledEntryFrame .ssROI -parent $w.fg.x1.y3 -label "Spot ROI Size (x, y): "  -orientation horizontal \
      -variableList "spotROISize(x,$group) spotROISize(y,$group)" \
      -width 4 -contextHelp "Enter the size of the ROI for spot analysis, in the order xSize, ySize.  This ROI is automatically centered on the spot."
    APSLabeledEntryFrame .sizelines -parent $w.fg.x2.y3 \
      -label "Lines for size analysis (x, y): " -orientation horizontal \
      -variableList "sizeLines(x,$group) sizeLines(y,$group)" \
      -width 4 -contextHelp "Enter the number of lines to use for size analysis.  The spot sizes will be computed using this number of lines for x and y, with the lines centered on the peak intensity."

    APSLabeledEntry .bgle -parent $w.fg.x1.y4 -label "Background determination half-width: " \
      -textVariable backgroundHalfWidth($group) -contextHelp \
      "The half-width, in intensity, of the intensities used for computation of the background.  The region is centered on the most populous intensity in the intensity histogram."
    APSLabeledEntry .slle -parent $w.fg.x2.y4 -label "Saturation level: " -textVariable saturationLevel($group) \
      -contextHelp "The level above which saturation is deemed to occur.  Used to determine the number of saturated pixels."

    APSRadioButtonFrame .symbg -parent $w.fg.x1.y5 -label "Symmetric BG removal: " -orientation horizontal \
      -variable symmetricBackgroundRemoval($group) \
      -buttonList "Yes No" -valueList "1 0" \
      -contextHelp "Following background subtraction, scans the spot ROI and sets to zero those pixels that are within the backgroundHalfWidth of 0.  This balances the setting of negative pixels to 0 and avoid introducing a positive bias into the image."
    APSRadioButtonFrame .loner -parent $w.fg.x2.y5 -label "Loner removal: " -orientation horizontal \
      -variable lonerRemoval($group) \
      -buttonList "Yes No" -valueList "1 0" \
      -contextHelp "If enabled, scans the spot ROI and sets loner pixels to zero.  A loner is a pixel whose eight neighbor pixels are all zero.  Loner pixels are unphysical and due to noise."

    APSLabeledEntry .le1 -parent $w.fg.x1.y6 -label "Center tolerance*: " \
      -textVariable centroidTolerance($group) -contextHelp \
      "Allowed deviation of center (in pixels) from mode of all centers.  A deviation outside this window means a point is bad.  Applied at collation stage."
    APSLabeledEntry .le2 -parent $w.fg.x2.y6 -label "Allowed sat. pixels*: " \
      -textVariable allowedSaturatedPixels($group) -contextHelp \
      "Number of saturated pixels allowed before an image is considered bad.  Applied at collation stage."

    APSLabeledOutput .numfiles -parent $w.fg.x1.y7 -label "Number of data files: " \
      -textVariable numberOfFiles($group) \
      -contextHelp "Shows the number of files selected for this group."

    APSFrame .brow1 -parent $w -label "" -relief flat -packOption "-side top -anchor w"
    set w1 $w.brow1.frame
    APSButton .pick -parent $w1 -text "Pick files..." -command \
      "SelectVLDFiles -group $group" -contextHelp "Pick VLD files for analysis using a scrolled list that permits selecting discontinous groups of files."
    APSButton .select -parent $w1 -text "Select range of files..." -command \
      "SelectVLDFiles -group $group -range 1" -contextHelp "Specify VLD files for analysis by giving the starting and ending index of the files."
    APSButton .list -parent $w1 -text "List selected files..." -command \
      "ListVLDFiles -group $group" -contextHelp "Displays a list of the VLD files selected for this group."
    APSButton .univ -parent $w1 -text "Universalize settings..." -command \
      "UniversalizeSettings -group $group" -contextHelp \
      "Copies the settings for this group to all other groups.  Doesn't copy filenames or ND level."
      
    APSFrame .brow2 -parent $w -label "" -relief flat -packOption "-side top -anchor w"
    set w1 $w.brow2.frame
    APSButton .dqs -parent $w1 -text "Process (DQS)" -command \
      "ProcessVLDFiles -group $group -mode batch" \
      -contextHelp "Process the files for this group using DQS.  Available from oxygen clients only.  Typically takes 5-10 minutes for 100 files."
    APSButton .dqsws -parent $w1 -text "Process w/Images (DQS)" -command \
      "ProcessVLDFiles -group $group -mode batch -withSpotImages 1" \
      -contextHelp "Process the files for this group using DQS.  Available from oxygen clients only.  Typically takes 5-10 minutes for 100 files."
    APSButton .proc -parent $w1 -text "Process" -command \
      "ProcessVLDFiles -group $group -mode local" \
      -contextHelp "Process the files for this group using the local workstation.  Typically takes 5-10 minutes for 100 files."
    APSButton .plot1 -parent $w1 -text "Plots vs. index" -command \
      "PlotAnalysisVsIndex -group $group" -contextHelp "Plot the results of analysis for this group.  You must process the data first."
    APSButton .plot2 -parent $w1 -text "Scatter plots" -command \
      "PlotAnalysisScatter -group $group" -contextHelp "Plot the results of analysis for this group.  You must process the data first."
    APSButton .plot4 -parent $w1 -text "Spot images" -command \
      "PlotSpotImages -group $group" -contextHelp "Plot the images of the beam spots, if available."

    APSFrame .brow3 -parent $w -label "" -relief flat -packOption "-side top -anchor w"
    set w1 $w.brow3.frame
    APSButton .plot1 -parent $w1 -text "Histograms" -command \
      "PlotAnalysisHistograms -group $group" -contextHelp "Plot histograms of the results for this group.  You must process the data first."
    APSButton .plot2 -parent $w1 -text "quickSDDSplot" -command \
      "RunQuickSDDSPlot -group $group" -contextHelp "Run quickSDDSplot script with the results for this group.  You must process the data first."

}

proc ConvertAllFilterWheelToFilterLevel {} {
    global VLDGroupList filterWheelFile
    foreach group $VLDGroupList {
        if ![string length $filterWheelFile($group)] continue
        if ![file exists $filterWheelFile($group)] {
            setStatus "VLD $group: not found: $filterWheelFile($group)"
        }
        if [catch {ConvertFilterWheelToFilterLevel -group $group} result] {
            setStatus "VLD $group: $group"
        }
    }
}

proc ConvertFilterWheelToFilterLevel {args} {
    set group -1
    APSStrictParseArguments {group}

    global NDFilter NDFilterWheel filterWheelFile
    set tmpFile /tmp/[APSTmpString]
    set VLD [os editstring 1f1d $group]
    set FilterWheelDataFile $filterWheelFile($group)
    if [catch {exec sddsprocess $FilterWheelDataFile $tmpFile -nowarning \
                 -match=column,FilterWheel=D${VLD}:VLD:F* \
                 "-test=param,n_rows 0 >" 
        sdds load $tmpFile FilterWheelData} result] {
        return -code error "$result"
    }
    set calibrated 0
    set totalAttenuation 0
    foreach wheel {1 2 3} {
        if {[string match open* [string tolower $NDFilterWheel($wheel,$group)]] || \
              [string match absent* [string tolower $NDFilterWheel($wheel,$group)]]} {
            continue
        }
        set textFW 0
        if [string match *\[A-Za-z\]* $NDFilterWheel($wheel,$group)] {
            set textFW 1
        } else {
            if $NDFilterWheel($wheel,$group)<0 continue
        }
        set PositionName [lindex $FilterWheelData(Column.PositionName) [expr $wheel-1]]
        set NominalAttenuation [lindex $FilterWheelData(Column.NominalAttenuation) [expr $wheel-1]]
        set MeasuredAttenuation [lindex $FilterWheelData(Column.MeasuredAttenuation) [expr $wheel-1]]
        set matchFound 0
        foreach PN $PositionName NA $NominalAttenuation MA $MeasuredAttenuation {
            if $textFW {
                if [string compare $PN $NDFilterWheel($wheel,$group)]==0 {
                    set totalAttenuation [expr $totalAttenuation+$MA]
                    set matchFound 1
                    break
                }
            } else {
                if [expr abs($NA-$NDFilterWheel($wheel,$group))<0.01] {
                    set totalAttenuation [expr $totalAttenuation+$MA]
                    set matchFound 1
                    break
                }
            }
        }
        if !$matchFound {
            return -code error "Error: no data for filter wheel #$wheel setting $NDFilterWheel($wheel,$group)"
            set calibrated 0
            break
        }
        set calibrated 1
    }
    if $calibrated {
        set NDFilter($group) $totalAttenuation
    }
}

proc PlotSpotImages {args} {
    set group 0
    APSStrictParseArguments {group}

    global dataDirectory numberOfFiles listOfRootnames  experimentName workDirectory

    set spotFile $workDirectory/$experimentName.VLD$group.spots
    if ![file exists $spotFile] {
        setStatus "Not found: $spotFile"
        return
    }

    exec sddscontour -shade=256,0,255 $spotFile &
    setStatus "Images plotting.  Please stand by."
}


proc PlotAnalysisVsIndex {args} {
    set group 0
    APSStrictParseArguments {group}

    global dataDirectory numberOfFiles listOfRootnames  experimentName workDirectory
    
    set procFile $workDirectory/$experimentName.VLD$group.proc
    if ![file exists $procFile] {
        setStatus "Not found: $procFile"
        return
    }
    if {[catch {exec sdds2stream -rows=bare $procFile} rows] || \
          ![string length $rows] || $rows!=$numberOfFiles($group)} {
        setStatus "Processing not complete."
        return
    }
    exec sddsplot \
      "-topline=$procFile" \
      -column=FileIndex,SaturationCount \
      -column=FileIndex,x* -column=FileIndex,y* \
      -column=FileIndex,BackgroundLevel -column=FileIndex,IntegratedSpotIntensity \
      -column=FileIndex,PeakSpotIntensity $procFile -sep &
}

proc RunQuickSDDSPlot {args} {
    set group 0
    APSStrictParseArguments {group}

    global dataDirectory numberOfFiles listOfRootnames  experimentName workDirectory
    
    set procFile $workDirectory/$experimentName.VLD$group.proc
    if ![file exists $procFile] {
        setStatus "Not found: $procFile"
        return
    }
    if {[catch {exec sdds2stream -rows=bare $procFile} rows] || \
          ![string length $rows] || $rows!=$numberOfFiles($group)} {
        setStatus "Processing not complete."
        return
    }
    exec quickSDDSplot -dataFileList $procFile &

}

proc PlotAnalysisHistograms {args} {
    set group 0
    APSStrictParseArguments {group}

    global dataDirectory numberOfFiles listOfRootnames  experimentName workDirectory
    global NDFilter
    scan $group %ld VLD

    set procFile $workDirectory/$experimentName.VLD$group.proc
    if ![file exists $procFile] {
        setStatus "Not found: $procFile"
        return
    }
    if {[catch {exec sdds2stream -rows=bare $procFile} rows] || \
          ![string length $rows] || $rows!=$numberOfFiles($group)} {
        setStatus "Processing not complete."
        return
    }
    set bins [expr $rows/5]
    if $bins<3 {
        set bins 3
    }
    set tmpFile /tmp/[APSTmpString]
    APSAddToTempFileList $tmpFile

    if [catch {exec sddsprocess $procFile -pipe=out \
                  "-define=column,AdjustedIntensity,IntegratedSpotIntensity 10 $NDFilter($VLD) pow *" \
                  | sddsmultihist -pipe=in $tmpFile -separate -bins=$bins -sides \
                 -columns=AdjustedIntensity,IntegratedSpotIntensity,x*,y*} result] {
        setStatus "$result"
        return
    }
    if [catch {exec sddsquery $tmpFile -column | fgrep Frequency | editstring -stream -edit=%/Frequency//} \
          columnList] {
        setStatus "$columnList"
        return
    }
    set optList ""
    foreach column $columnList {
        lappend optList -column=$column,${column}Frequency 
        if [string compare $column AdjustedIntensity]==0 {
            lappend optList "-title=ND Filter: $NDFilter($VLD)"
        }
    }
    eval exec sddsplot {"-topline=$procFile"} \
      $tmpFile -separate $optList &
}

proc PlotAnalysisScatter {args} {
    set group 0
    APSStrictParseArguments {group}

    global dataDirectory numberOfFiles listOfRootnames  experimentName workDirectory
    
    set procFile $workDirectory/$experimentName.VLD$group.proc
    if ![file exists $procFile] {
        setStatus "Not found: $procFile"
        return
    }
    if {[catch {exec sdds2stream -rows=bare $procFile} rows] || \
          ![string length $rows] || $rows!=$numberOfFiles($group)} {
        setStatus "Processing not complete."
        return
    }
    exec sddsplot \
        "-topline=$procFile" \
        -column=IntegratedSpotIntensity,(x*,y*) $procFile -sep -graph=dot &
}

proc ProcessVLDFiles {args} {
    set group 0
    set settings ""
    set mode batch
    set withSpotImages 0
    APSStrictParseArguments {group mode settings withSpotImages}
    global experimentName backgroundHalfWidth lonerRemoval symmetricBackgroundRemoval
    global ROI spotROISize sizeLines blankOut workDirectory
    set oldDir [pwd]
    cd $workDirectory
    if {![string length $settings]} {
        set settings $experimentName.VLD$group.settings
    }
    if {![file exists $settings]} {
        cd $oldDir
        return -code error "Configuration file ($settings) not found."
    }
    eval file delete -force $experimentName.allproc
    set rootname $experimentName.VLD$group
    eval file delete -force $rootname.proc $rootname.series $rootname.sum $rootname.spots
    set spotImageArg ""
    if $withSpotImages {
        set spotImageArg "-spotImages $rootname.spots"
    }
    switch $mode {
        batch {
            setStatus "Submitting job for VLD $group using $settings..."
            catch {eval exec echo "runProcTcl ProcessVLDData \
                     -output $experimentName.VLD$group.proc \
                     -ROISizex $spotROISize(x,$group) -ROISizey $spotROISize(y,$group) -group $group \
                     -ROIx0 $ROI(x0,$group) -ROIy0 $ROI(y0,$group) \
                     -ROIx1 $ROI(x1,$group) -ROIy1 $ROI(y1,$group) \
                     -blankOutx0 $blankOut(x0,$group) -blankOuty0 $blankOut(y0,$group) \
                     -blankOutx1 $blankOut(x1,$group) -blankOuty1 $blankOut(y1,$group) \
                     -sizeLinesx $sizeLines(x,$group) \
                     -sizeLinesy $sizeLines(y,$group) \
                     -backgroundHalfWidth $backgroundHalfWidth($group) \
                     -lonerRemoval $lonerRemoval($group) \
                     -symmetricBackgroundRemoval $symmetricBackgroundRemoval($group) \
                     -configFile $settings $spotImageArg" \
                     | qsub -cwd -j y -N V${group}$experimentName} result
            setStatus $result
            set jobID [lindex [split $result] 2]
            after 10000 "CheckDQSJob -directory $workDirectory -jobID $jobID -group $group -experimentName $experimentName"
        }
        default {
            setStatus "Processing VLD $group using $settings..."
            if [catch {eval ProcessVLDData -output $experimentName.VLD$group.proc \
                         -ROISizex $spotROISize(x,$group) -ROISizey $spotROISize(y,$group) -group $group \
                         -ROIx0 $ROI(x0,$group) -ROIy0 $ROI(y0,$group) \
                         -ROIx1 $ROI(x1,$group) -ROIy1 $ROI(y1,$group) \
                         -blankOutx0 $blankOut(x0,$group) -blankOuty0 $blankOut(y0,$group) \
                         -blankOutx1 $blankOut(x1,$group) -blankOuty1 $blankOut(y1,$group) \
                         -sizeLinesx $sizeLines(x,$group) \
                         -sizeLinesy $sizeLines(y,$group) \
                         -backgroundHalfWidth $backgroundHalfWidth($group) \
                         -lonerRemoval $lonerRemoval($group) \
                         -symmetricBackgroundRemoval $symmetricBackgroundRemoval($group) \
                         -configFile $settings -spotImages $spotImageFile} result]] {
                setStatus $result
            }
            setStatus "Done."
        }
    }
    cd $oldDir
}

proc UniversalizeSettings {args} {
    set group 0
    APSStrictParseArguments {group}

    global ROI spotROISize sizeLines
    global ROI blankOut backgroundHalfWidth lonerRemoval
    global VLDGroupList symmetricBackgroundRemoval
    global saturationLevel centroidTolerance allowedSaturatedPixels filterWheelFile

    global doUniversalize
    set doUniversalize(ROI) 0
    set doUniversalize(blankOut) 0
    set doUniversalize(spotROISize) 0
    set doUniversalize(sizeLines) 0
    set doUniversalize(backgroundHalfWidth) 0
    set doUniversalize(lonerRemoval) 0
    set doUniversalize(symmetricBackgroundRemoval) 0
    set doUniversalize(saturationLevel) 0
    set doUniversalize(centroidTolerance) 0
    set doUniversalize(allowedSaturatedPixels) 0
    set doUniversalize(filterWheelFile) 0
    set doUniversalize(cancel) -1
    set w [APSUniqueName .]
    APSDialogBox $w  -name "Universalize" \
      -cancelCommand "set doUniversalize(cancel) 1" \
      -okCommand "set doUniversalize(cancel) 0" 
    APSCheckButtonFrame .cb -parent $w.userFrame \
      -label "Choose items:" -orientation vertical \
      -buttonList "ROI BlankOutRegion SpotROISize SizeLines BackgroundHalfDidth LonerRemoval SymmetricBackgroundRemoval SaturtionLevel CentroidTolerance AllowedSaturatedPixels FilterWheelFile" \
      -variableList "doUniversalize(ROI) doUniversalize(blankOut) doUniversalize(spotROISize) \
doUniversalize(sizeLines) doUniversalize(backgroundHalfWidth) doUniversalize(lonerRemoval) \
doUniversalize(symmetricBackgroundRemoval) doUniversalize(saturationLevel) doUniversalize(centroidTolerance) doUniversalize(allowedSaturatedPixels) doUniversalize(filterWheelFile) " 
    tkwait variable doUniversalize(cancel)
    if $doUniversalize(cancel)==1 return

    foreach group0 $VLDGroupList {
        foreach type {x0 x1 y0 y1} {
            if $doUniversalize(ROI) {
                set ROI($type,$group0) $ROI($type,$group)
            }
            if $doUniversalize(blankOut) {
                set blankOut($type,$group0) $blankOut($type,$group)
            }
        }
        foreach type {x y} {
            if $doUniversalize(spotROISize) {
                set spotROISize($type,$group0) $spotROISize($type,$group)
            }
            if $doUniversalize(sizeLines) {
                set sizeLines($type,$group0) $sizeLines($type,$group)
            }
        }
        foreach field {backgroundHalfWidth lonerRemoval symmetricBackgroundRemoval saturationLevel centroidTolerance allowedSaturatedPixels filterWheelFile} {
            if $doUniversalize($field) {
                set $field\($group0\) [set $field\($group\)]
            }
        }
    }
    setStatus "Settings for group $group copied to other groups."
}


proc SelectVLDFiles {args} {
    set group 0
    set range 0
    APSStrictParseArguments {group range}
    global dataDirectory

    set VLD [os editstring 1f1d $group]
    set directory $dataDirectory/VLD$VLD
    if ![file exists $directory] {
        return -code error "Not found: $directory"
    }
    set sddsFileList [lsort [glob -nocomplain $directory/VLD${VLD}_*.sdds.gz]] 
    if ![llength $sddsFileList] {
        set sddsFileList [lsort [glob -nocomplain $directory/*.sdds.gz]]
    }
    set hdfFileList [lsort [glob -nocomplain $directory/VLD${VLD}_*.hdf]] 
    if ![llength $hdfFileList] {
        set hdfFileList [lsort [glob -nocomplain $directory/*.hdf]]
    }
    if ![llength $sddsFileList] {
        set fileList $hdfFileList
    } elseif ![llength $hdfFileList] {
        set fileList $sddsFileList
    } else {
        set fileList ""
        foreach file $hdfFileList {
            lappend fileList [os editstring %/.hdf// $file]
        }
        foreach file $sddsFileList {
            lappend fileList [os editstring %/.sdds.gz// $file]
        }
        set fileList0 [lsort $fileList]
        set fileList ""
        set lastFile ""
        foreach file $fileList0 {
            if [string compare $lastFile $file]==0 continue
            lappend fileList $file
            set lastFile $file
        }
    }

    if [llength $fileList]==0 {
        return -code error "No .hdf or .sdds.gz files files in $directory."
    }
    
    set itemList [eval os editstring 100Z/i.VLD$VLD/.%/.hdf//%/.sdds.gz// $fileList]
    set rootnameList [eval os editstring %/.hdf//%/.sdds.gz// $fileList]
    global listOfRootnames numberOfFiles 
    if $range {
        set db [APSUniqueName .]
        global fileIndex1 fileIndex2 rangeDialogDone
        APSDialogBox $db -okCommand "set rangeDialogDone ok" \
          -cancelCommand "set rangeDialogDone cancel" -name "Group $group selection"
        set fileIndex1 -1
        set fileIndex2 -1
        APSLabeledEntry .le1 -parent $db.userFrame -label "First index: " -textVariable fileIndex1
        APSLabeledEntry .le2 -parent $db.userFrame -label "Last  index: " -textVariable fileIndex2
        set rangeDialogDone ""
        tkwait variable rangeDialogDone
        if {[string compare $rangeDialogDone cancel]==0 || $fileIndex1==-1 || $fileIndex2==-1} {
            setStatus "File selection cancelled."
            return
        }
        if ($fileIndex2<$fileIndex1) {
            setStatus "Error: Invalid indices"
            return
        }
        for {set index $fileIndex1} {$index<=$fileIndex2} {incr index} {
            set trialName VLD$VLD/VLD${VLD}_[format %010ld $index]
            if [set position [lsearch -exact $itemList $trialName]]==-1 {
                setStatus "Error: index $index has no file."
                return
            }
            lappend chosenList [lindex $rootnameList $position]
        }
    } else {
        set chosenList [APSChooseItemFromList \
                          -name "Group $group selection" \
                          -itemList $itemList \
                          -returnList $rootnameList \
                          -returnIndices 0 \
                          -multiItem 1 \
                          -contextHelp "Choose one or more files for inclusion in analysis"]
    }
    if [llength $chosenList]==0 {
        return
    }
    set listOfRootnames($group) $chosenList
    set numberOfFiles($group) [llength $chosenList]
    update
}


proc ListVLDFiles {args} {
    set group 0
    APSStrictParseArguments {group}
    global listOfRootnames numberOfFiles 
    if !$numberOfFiles($group) {
        setStatus "No files selected.  (Did you load the configuration, if necessary?)"
        return
    }
    set junk [APSChooseItemFromList \
                -name "Group $group selections" \
                -itemList $listOfRootnames($group) \
                -returnIndices 0 \
                -contextHelp "Shows which files are selected for group $group."]
}

proc BuildApplication {} {
    global experimentName dataDirectory status VLDGroupList workDirectory plotWithColor

    APSApplication . -name ProcessVLDData \
    -overview "Processes VLD data in HDF files to give intensity as a function of VLD number."
    
    set status "N.B.: items marked with * are implemented at the collation stage."
    APSScrolledStatus .status -parent .userFrame -textVariable status -height 10 -width 80
    set status "All others require reprocessing the images to take effect."
    update

    foreach group $VLDGroupList {
        lappend labelList "Group $group"
        lappend widgetList .userFrame.wsf.frame.vld$group
    }
    APSWidgetSwapFrame .wsf -parent .userFrame -label "" \
        -labelList $labelList -widgetList $widgetList -height 385 -width 425 \
        -contextHelp "Press buttons below to switch to a different group."
    pack propagate .userFrame.wsf.frame 0
    pack .userFrame.wsf -side top

    foreach group $VLDGroupList {
        MakeVLDWidgets .vld$group -parent .userFrame.wsf.frame -group $group
        APSSwapInWidget .wsf -parent .userFrame -swapIn .userFrame.wsf.frame.vld$group
    }
    APSSwapInWidget .wsf -parent .userFrame -swapIn .userFrame.wsf.frame.vld9

    APSLabeledEntry .le3 -parent .userFrame -textVariable workDirectory \
    -label "Work directory: " -width 60 -contextHelp "Enter the name of the directory in which to place processed results."

    APSLabeledEntry .le1 -parent .userFrame -textVariable experimentName \
    -label "Experiment name:" -width 60 -contextHelp "Enter the experiment name.  This will be used as the rootname for intermediate and final results files."
        
    APSLabeledEntry .le2 -parent .userFrame -textVariable dataDirectory \
    -label "Data directory: " -width 60 -contextHelp "Enter the name of the directory within which the VLD* subdiretories reside.  Note that if VLD data is still in HDF format, you'll need write access to this directory!"

    APSRadioButtonFrame .rb0 -parent .userFrame -variable plotWithColor -label "Plot type: " \
      -buttonList "B&W Color" -valueList "0 1"  -orientation horizontal

    APSFrame .brow0 -parent .userFrame -relief flat -label "" -packOption "-side top -anchor w"
    set w .userFrame.brow0.frame
    APSButton .computefilters -parent $w -text "All filter wheels -> ND value" \
      -command "ConvertAllFilterWheelToFilterLevel" -contextHelp \
      "Converts filter wheel data to filter level for all stations."
    APSButton .readcamcal -parent $w -text "Read camera sensitivities" \
      -command "ReadCameraSensitivityData" -contextHelp \
      "Reads an SDDS file (specified by user) giving sensitiviy data for the cameras."

    APSFrame .brow1 -parent .userFrame -relief flat -label "" -packOption "-side top -anchor w"
    set w .userFrame.brow1.frame
    APSButton .resetconfig -parent $w -text "Reset Configuration" -command \
        "ResetVLDConfig -confirm 1" -contextHelp "Resets the configuration of the VLDs for an experiment.  In particular, all the settings are restored to the defaults and the file lists are erased."
    APSButton .saveconfig -parent $w -text "Save Configuration" -command \
        SaveVLDConfig -contextHelp "Saves the configuration of the VLDs for an experiment.  In particular, saves the list of files, ND filter setting, ROI settings, etc. for each VLD."
    APSButton .loadconfig -parent $w -text "Load Configuration" -command \
        LoadVLDConfig -contextHelp "Loads the configuration of the VLDs for an experiment.  In particular, loads the list of files, ND filter setting, ROI settings, etc. for each VLD.  You must first save a configuration with this tool in order to load it later."
    APSButton .process -parent $w -text "Process All (DQS)" -command \
      ProcessAllDQS -contextHelp "Processes images for all VLDs using DQS jobs."

    APSFrame .brow2 -parent .userFrame -relief flat -label "" -packOption "-side top -anchor w"
    set w .userFrame.brow2.frame
    APSButton .processs -parent $w -text "Process All w/ Spot Images (DQS)" -command \
      "ProcessAllDQS -withSpotImages 1 " -contextHelp "Processes images for all VLDs using DQS jobs and makes spot image files."
    APSButton .check -parent $w -text "Check Data" -command \
      "CheckProcessedVLDData" -contextHelp \
      "Checks for the existence of processed data for this data set."
    APSButton .collate -parent $w -text "Collate Data" -command \
        "CollateProcessedVLDData" -contextHelp \
        "Collates intensity data for all selected VLDs that have processed image data available.  The ND factors are applied at this stage."
    APSButton .recollate -parent $w -text "Recollate All Data" -command \
        RecollateAll -contextHelp \
        "Recollates all intensity data for all selected VLDs that have processed image data available.  The ND factors are applied at this stage."
    APSButton .plot -parent $w -text "Plot Results" -command \
        "PlotProcessedVLDData" -contextHelp \
        "Plots intensity vs VLD number."

    APSFrame .brow3 -parent .userFrame -relief flat -label "" -packOption "-side top -anchor w"
    set w .userFrame.brow3.frame
    APSButton .fit -parent $w -text "Gain fit..." -command \
      "PerformGainFit" -contextHelp "Fits an exponential to provide the gain length."
    APSButton .quickPlot -parent $w -text "quickSDDSplot..." -command LaunchQuickSDDSPlot  \
      -contextHelp "Launches quickSDDSplot with the processed data file."
    APSButton .export -parent $w -text "Export..." -command LaunchExportData \
      -contextHelp "Launches sddsExportData with the processed data file."
}

proc  LaunchExportData {} {
    global numberOfFiles listOfRootnames NDFilter experimentName dataDirectory workDirectory
    if ![file exists $workDirectory/$experimentName.allproc] {
        return -code error "Not found: $workDirectory/$experimentName.allproc"
    }
    exec sddsExportData -dataDirectory $workDirectory -dataFileList $experimentName.allproc \
      -message "$experimentName from $workDirectory"  \
      -timeFilter 0
}

proc  LaunchQuickSDDSPlot {} {
    global numberOfFiles listOfRootnames NDFilter experimentName dataDirectory workDirectory
    if ![file exists $workDirectory/$experimentName.allproc] {
        return -code error "Not found: $workDirectory/$experimentName.allproc"
    }
    exec quickSDDSplot -dataFileList $workDirectory/$experimentName.allproc &
}

proc PerformGainFit {} {
    global numberOfFiles listOfRootnames NDFilter experimentName dataDirectory workDirectory
    set oldDir [pwd]
    cd $workDirectory
    if ![file exists $experimentName.allproc] {
        cd $oldDir
        return -code error "Not found: $experimentName.allproc"
    }
    set w [APSUniqueName .]
    set fitResponse -1
    APSDialogBox $w -name "Fit dialog" -cancelCommand "set fitResponse 0" \
      -okCommand "set fitResponse 1"
    set w $w.userFrame
    global firstFitVLD lastFitVLD fitForm
    set firstFitVLD 1
    set lastFitVLD 9
    set fitForm 0
    APSLabeledEntry .le1 -parent $w -label "First VLD: " -textVariable firstFitVLD 
    APSLabeledEntry .le2 -parent $w -label "Last VLD: " -textVariable lastFitVLD 
    set fitFormList {"exp(z/L)" "exp(z/L)/z"}
    APSRadioButtonFrame .eq -parent $w -label "Equation: " \
      -variable fitForm -buttonList $fitFormList \
      -valueList {0 1} 
    tkwait variable fitResponse
    if !$fitResponse {
        cd $oldDir
        return
    }
    if $firstFitVLD>=$lastFitVLD {
        setStatus "Invalid VLD range"
        cd $oldDir
        return
    }
    set listOfTypes [exec sddsquery -column -sdds $experimentName.allproc \
                       | sddsprocess -pipe -match=column,Name=AdjustedIntensity*Ave \
                       -reedit=column,Name,%/AdjustedIntensity// \
                       | sdds2stream -pipe -column=Name]
    foreach type $listOfTypes {
        if [catch {exec sddsprocess $experimentName.allproc -pipe=out \
                     -filter=col,VLD,[expr $firstFitVLD-0.5],[expr $lastFitVLD+0.5] \
                     "-define=column,z,VLD 2.4 *,units=m" \
                     -process=z,min,z0 \
                     "-define=column,FitFactor,$fitForm 0 == ? 1 : z z0 - 2.4 + \$ " \
                     "-define=column,%sLn,%s FitFactor * ln,select=AdjustedIntensity${type}" \
                     | sddspfit -terms=2 -generate -pipe \
                     -column=z,AdjustedIntensity${type}Ln \
                     | sddsprocess -pipe=in /tmp/$experimentName.fit.$type \
                     "-define=parameter,GainLength,1 Slope /,units=m" \
                     "-define=parameter,GainLengthSigma,SlopeSigma Slope sqr /,units=m" \
                     "-print=parameter,GainLengthLabel,L\$bg\$n = %.2f +/- %.2f m,GainLength,GainLengthSigma" \
                 } result] {
            cd $oldDir
            return -code error "$result"
        }
        lappend fileList /tmp/$experimentName.fit.$type 
        lappend plotOptions -column=z,AdjustedIntensity${type}Ln -graph=sym,scale=2 
        lappend plotOptions /tmp/$experimentName.fit.$type "-topline=Fit to $type results" 
        lappend plotOptions -column=z,AdjustedIntensity${type}LnFit 
        lappend plotOptions /tmp/$experimentName.fit.$type -end
    }
    eval exec sddsplot -same=y,global $plotOptions &
    eval exec sddscombine $fileList -collapse -pipe=out \
      | sddsprocess -pipe \
      -edit=column,DataType,Filename,10Z. \
      | sddsprintout -pipe=in /tmp/$experimentName.fit.printout \
      {"-title=Fit results for $experimentName from $workDirectory\nVLDs $firstFitVLD to $lastFitVLD\nFit to form [lindex $fitFormList $fitForm]\n\n"} \
      -column=DataType -column=GainLength,format=%20.3f -column=GainLengthSigma,format=%20.3f
    APSFileDisplayWindow [APSUniqueName .] -fileName /tmp/$experimentName.fit.printout 
}


proc ProcessAllDQS {args} {
    set withSpotImages 0
    APSStrictParseArguments {withSpotImages}

    global VLDGroupList experimentName workDirectory
    APSFreezeVars experimentName workDirectory
    set answer [APSMultipleChoice [APSUniqueName .] -question \
                  "Really?  Process all files?" \
                  -labelList "Yes No" -returnList "1 0"]
    if $answer==0 {
        APSUnfreezeVars experimentName workDirectory
        return
    }
    foreach group $VLDGroupList {
        if [file exists $workDirectory/$experimentName.VLD$group.settings] {
            ProcessVLDFiles -group $group -settings $experimentName.VLD$group.settings \
              -mode batch -withSpotImages $withSpotImages
        }
    }
    APSUnfreezeVars experimentName workDirectory
}

proc CheckProcessedVLDData {} {
    global numberOfFiles listOfRootnames NDFilter experimentName dataDirectory VLDGroupList
    global centroidTolerance allowedSaturatedPixels
    global workDirectory 
    LoadVLDConfig
    APSFreezeVars experimentName dataDirectory workDirectory
    setStatus "Checking data for experiment $experimentName..."
    set resultList ""
    set oldDir [pwd]
    cd $workDirectory
    set tmpRoot /tmp/[APSTmpString]
    foreach group $VLDGroupList {
        if !$numberOfFiles($group) {
            continue
        }
        if ![file exists $experimentName.VLD$group.proc] {
            setStatus "Warning: $experimentName.VLD$group.proc not found"
            continue
        }
        if [catch {exec sdds2stream -rows=bare $experimentName.VLD$group.proc} rows] {
            setStatus "Warning: $experimentName.VLD$group.proc is not a valid SDDS file (still processing?)"
            continue
        }
        if [string compare $rows $numberOfFiles($group)]!=0 {
            setStatus "Warning: $experimentName.VLD$group.proc doesn't have the expected number of rows (still processing?)."
            continue
        }
    }
    APSUnfreezeVars experimentName dataDirectory workDirectory
    setStatus "Done."
}

proc RecollateAll {} {
    global experimentName workDirectory
    set oldDir [pwd]
    cd $workDirectory
    set experimentList [eval os editstring %/.allproc// [lsort [glob -nocomplain *.allproc]]]
    cd $oldDir
    foreach experimentName $experimentList {
        CollateProcessedVLDData
    }
}

proc CollateProcessedVLDData {} {
    global numberOfFiles listOfRootnames NDFilter experimentName dataDirectory VLDGroupList
    global centroidTolerance allowedSaturatedPixels
    global workDirectory cameraSensitivity
    LoadVLDConfig
    APSFreezeVars experimentName dataDirectory workDirectory
    setStatus "Collating data for experiment $experimentName..."
    set resultList ""
    set oldDir [pwd]
    cd $workDirectory
    set tmpRoot /tmp/[APSTmpString]
    foreach group $VLDGroupList {
        if !$numberOfFiles($group) {
            setStatus "Warning: no files chosen for VLD $group"
            continue
        }
        if ![file exists $experimentName.VLD$group.proc] {
            setStatus "Warning: $experimentName.VLD$group.proc not found"
            continue
        }
        if [catch {exec sdds2stream -rows=bare $experimentName.VLD$group.proc} rows] {
            setStatus "Warning: $experimentName.VLD$group.proc is not a valid SDDS file (still processing?)"
            continue
        }
        if [string compare $rows $numberOfFiles($group)]!=0 {
            setStatus "Warning: $experimentName.VLD$group.proc doesn't have the expected number of rows (still processing?)."
            continue
        }
        set nTop [expr int($numberOfFiles($group)*0.05+0.5)]
        if $nTop==0 {
            set nTop 1
        }
        set VLD [os editstring 1f1d $group]
        if [catch {eval exec sddsprocess $experimentName.VLD$group.proc -pipe=out \
                     {"-define=parameter,NDSetting,$NDFilter($group)"} \
                     {"-define=parameter,CameraSensitivity,$cameraSensitivity($group)"} \
                     {"-define=column,AdjustedIntensity,IntegratedSpotIntensity 10 NDSetting pow * CameraSensitivity /"} \
                     {"-define=column,IsSaturated,SaturationCount $allowedSaturatedPixels($group) > ? 1 : 0 \$,type=long"} \
                     {"-define=column,VLD,$VLD,type=short"} \
                     | sddssort -pipe -col=AdjustedIntensity,decr \
                     | tee $experimentName.VLD$group.series \
                     | sddsprocess -pipe \
                     -process=IsSaturated,sum,NumberSaturated \
                     {"-define=parameter,FractionSaturated,NumberSaturated n_rows /"} \
                     -process=BackgroundLevel,ave,%sAve \
                     -process=SaturationCount,max,%sMax \
                     -process=SaturationCount,ave,%sAve \
                     -process=AdjustedIntensity,ave,%sAve \
                     -process=AdjustedIntensity,stand,%sStDev \
                     -process=AdjustedIntensity,sigma,%sSigma \
                     -process=AdjustedIntensity,drange,%sDRange \
                     -process=AdjustedIntensity,qrange,%sQRange \
                     -process=*Cent*r*,ave,%sAve -process=*Range*,ave,%sAve \
                     -process=*Sigma*,ave,%sAve \
                     -process=*Cent*r*,stand,%sStDev \
                     -process=*Range*,stand,%sStDev \
                     -process=*Sigma*,stand,%sStDev \
                     -process=*Cent*r*,sigma,%sSigma \
                     -process=*Range*,sigma,%sSigma \
                     -process=*Sigma*,sigma,%sSigma \
                     -process=AdjustedIntensity,percentile,%sP90,percent=90 \
                     -process=AdjustedIntensity,percentile,%sP80,percent=80 \
                     -process=AdjustedIntensity,percentile,%sP75,percent=75 \
                     -process=AdjustedIntensity,percentile,%sP70,percent=70 \
                     -process=AdjustedIntensity,percentile,%sP60,percent=60 \
                     -process=AdjustedIntensity,percentile,%sP50,percent=50 \
                     -process=AdjustedIntensity,percentile,%sP40,percent=40 \
                     -process=AdjustedIntensity,percentile,%sP30,percent=30 \
                     -process=AdjustedIntensity,percentile,%sP25,percent=25 \
                     -process=AdjustedIntensity,percentile,%sP20,percent=20 \
                     -process=AdjustedIntensity,percentile,%sP10,percent=10 \
                     | sddsprocess -pipe -clip=$nTop,0,invert \
                     -process=IsSaturated,sum,NumberSaturatedTop \
                     {"-define=parameter,FractionSaturatedTop,NumberSaturatedTop n_rows /"} \
                     -process=BackgroundLevel,ave,%sTopAve \
                     -process=SaturationCount,ave,%sTopAve \
                     -process=SaturationCount,max,%sTopMax \
                     -process=AdjustedIntensity,ave,%sTopAve \
                     -process=AdjustedIntensity,stand,%sTopStDev \
                     -process=AdjustedIntensity,sigma,%sTopSigma \
                     -process=AdjustedIntensity,drange,%sTopDRange \
                     -process=AdjustedIntensity,qrange,%sTopQRange \
                     -define=parameter,VLD,$VLD,type=long \
                     -process=*Cent*r*,Ave,%sTopAve -process=*Range*,Ave,%sTopAve \
                     -process=*Sigma*,Ave,%sTopAve \
                     -process=*Cent*r*,stand,%sTopStDev -process=*Range*,stand,%sTopStDev \
                     -process=*Sigma*,stand,%sTopStDev \
                     -process=*Cent*r*,sigma,%sTopSigma -process=*Range*,sigma,%sTopSigma \
                     -process=*Sigma*,sigma,%sTopSigma \
                     | sddscollapse -pipe=in $experimentName.VLD$group.sum0} result] {
            file delete -force $experimentName.VLD$group.sum0
            setStatus "Problem collating data for VLD $group: $result"
            setStatus "This data is probably of poor quality and will not be used."
            continue
        }
        set fileList ""
        foreach level {5 10 20 30 40 100} {
            set perc [expr $level/100.0]
            if [catch {exec sddsprocess $experimentName.VLD$group.series -pipe=out \
                         -process=?SpotCenter,mode,%sMode,binsize=1 \
                         -process=BackgroundLevel,mode,%sMode,binsize=1 \
                         -filter=column,SaturationCount,-0.5,[expr $allowedSaturatedPixels($group)+0.5] \
                         "-define=col,%sOffset,%s %sMode -,select=?SpotCenter" \
                         "-define=col,%sOffset,%s %sMode -,select=BackgroundLevel" \
                         | sddsoutlier -pipe -absLimit=$centroidTolerance($group) -column=?SpotCenterOffset \
                         | sddsoutlier -pipe -absLimit=2  -column=BackgroundLevelOffset  \
                         | sddsprocess -pipe -nowarning \
                         "-test=column,i_row n_rows $perc * < " \
                         -process=AdjustedIntensity,count,NumberGood${level} \
                         -process=AdjustedIntensity,ave,%sGood${level}Ave \
                         -process=AdjustedIntensity,stand,%sGood${level}StDev \
                         -process=AdjustedIntensity,sigma,%sGood${level}Sigma \
                         -process=AdjustedIntensity,drange,%sGood${level}DRange \
                         -process=AdjustedIntensity,qrange,%sGood${level}QRange \
                         -define=parameter,VLD,$VLD,type=long \
                         -process=*Cent*r*,Ave,%sGood${level}Ave -process=*Range*,Ave,%sGood${level}Ave \
                         -process=*Sigma*,Ave,%sGood${level}Ave \
                         -process=*Cent*r*,stand,%sGood${level}StDev -process=*Range*,stand,%sGood${level}StDev \
                         -process=*Sigma*,stand,%sGood${level}StDev \
                         -process=*Cent*r*,sigma,%sGood${level}Sigma -process=*Range*,sigma,%sGood${level}Sigma \
                         -process=*Sigma*,sigma,%sGood${level}Sigma \
                         | sddscollapse -pipe=in $tmpRoot.$level} result] {
                setStatus "Problem collating data for VLD $group: $result"
                setStatus "This data is probably of poor quality and will not be used."
                continue
            }
            lappend fileList $tmpRoot.$level
        }
        if [catch {exec sddsprocess $experimentName.VLD$group.series -pipe=out \
                     -process=?SpotCenter,mode,%sMode,binsize=1 \
                     -process=BackgroundLevel,mode,%sMode,binsize=1 \
                     -filter=column,SaturationCount,-0.5,[expr $allowedSaturatedPixels($group)+0.5] \
                     "-define=col,%sOffset,%s %sMode -,select=?SpotCenter" \
                     "-define=col,%sOffset,%s %sMode -,select=BackgroundLevel" \
                     | sddsoutlier -pipe -absLimit=$centroidTolerance($group) -column=?SpotCenterOffset \
                     | sddsoutlier -pipe -absLimit=2  -column=BackgroundLevelOffset  \
                     | sddsprocess -pipe -nowarning \
                     -process=AdjustedIntensity,percentile,%sGoodP90,percent=90 \
                     -process=AdjustedIntensity,percentile,%sGoodP80,percent=80 \
                     -process=AdjustedIntensity,percentile,%sGoodP75,percent=75 \
                     -process=AdjustedIntensity,percentile,%sGoodP70,percent=70 \
                     -process=AdjustedIntensity,percentile,%sGoodP60,percent=60 \
                     -process=AdjustedIntensity,percentile,%sGoodP50,percent=50 \
                     -process=AdjustedIntensity,percentile,%sGoodP40,percent=40 \
                     -process=AdjustedIntensity,percentile,%sGoodP30,percent=30 \
                     -process=AdjustedIntensity,percentile,%sGoodP25,percent=25 \
                     -process=AdjustedIntensity,percentile,%sGoodP20,percent=20 \
                     -process=AdjustedIntensity,percentile,%sGoodP10,percent=10 \
                     -define=parameter,VLD,$VLD,type=long \
                     | sddscollapse -pipe=in $tmpRoot.perc} result] {
        }
        lappend fileList $tmpRoot.perc
        if [catch {eval exec sddsxref $fileList $experimentName.VLD$group.sum0 \
                     $experimentName.VLD$group.sum} result] {
            eval file delete -force $experimentName.VLD$group.sum0 $fileList
        } else {
            lappend resultList $experimentName.VLD$group.sum
        }
    }
    if ![llength $resultList] {
        APSUnfreezeVars  experimentName dataDirectory workDirectory
        setStatus "No data found!" 
        cd $oldDir
        return
    }
    setStatus "Done."
    eval exec sddscombine $resultList -merge -pipe=out \
    | sddssort -pipe -column=VLD,incr \
    | sddsprocess -pipe=in $experimentName.allproc \
      -process=Adjusted*Ave,first,%sFirst \
      {"-define=column,%sNorm,%s %sFirst /,select=Adjusted*Ave"} \
      {"-define=column,%sStDevNorm,%sStDev %sAve /,select=Adjusted*Ave,edit=%/Ave//"} \
      {"-define=column,%sDRangeNorm,%sDRange %sAve /,select=Adjusted*Ave,edit=%/Ave//"} \
      {"-define=column,%sQRangeNorm,%sQRange %sAve /,select=Adjusted*Ave,edit=%/Ave//"}
    PlotProcessedVLDData
    cd $oldDir
    APSUnfreezeVars  experimentName dataDirectory workDirectory
}

proc PlotProcessedVLDData {} {
    global numberOfFiles listOfRootnames NDFilter experimentName dataDirectory workDirectory plotWithColor
    set oldDir [pwd]
    cd $workDirectory
    if ![file exists $experimentName.allproc] {
        return -code error "Not found: $experimentName.allproc"
    }
    set fileList [glob -nocomplain $experimentName.VLD*.series]
    if [llength $fileList] {
        eval exec sddsplot $fileList -graph=sym,scale=0.5 \
          {"-topline=$experimentName from $workDirectory"} \
          -column=VLD,AdjustedIntensity -mode=y=log,y=special -range=ymin=1 &
    }
    if $plotWithColor {
        set varyOpt vary=type,subtype=type
    } else {
        set varyOpt vary=type
    }

    exec sddsplot "-topline=$experimentName from $workDirectory" \
      -graph=symbol,$varyOpt,scale=2 $experimentName.allproc \
      -column=VLD,FractionSaturated \
      "-ylabel=Fraction Saturated" "-legend=spec=All" \
      -column=VLD,FractionSaturatedTop  "-legend=spec=Top" -end \
      -column=VLD,SaturationCountMax  \
      "-ylabel=Number of saturated pixels" "-legend=spec=Max All" \
      -column=VLD,SaturationCountAve  "-legend=spec=Ave. All" \
      -column=VLD,SaturationCountTopMax "-legend=spec=Max Top" \
      -column=VLD,SaturationCountTopAve "-legend=spec=Ave. Top" -end \
      -column=VLD,AdjustedIntensityAve "-legend=spec=Av. All" -mode=y=log,y=special -range=ymin=1 \
      "-title=Intensity adjusted for ND filters" \
      "-ylabel=ND-Adjusted Intensity" \
      -column=VLD,AdjustedIntensityGood5Ave \
            "-legend=spec=Av. Good 5%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood10Ave \
            "-legend=spec=Av. Good 10%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood20Ave \
            "-legend=spec=Av. Good 20%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood100Ave \
            "-legend=spec=Av. Good 100%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityTopAve \
            "-legend=spec=Av. Top 5%" -range=ymin=1 -mode=y=log,y=special -end \
      -column=VLD,AdjustedIntensityAveNorm "-legend=spec=Av. All" -range=ymin=1 -mode=y=log,y=special \
      "-title=Intensity adjusted for ND filters and normalized to first VLD" \
      "-ylabel=Normalized Adjusted Intensity" \
      -column=VLD,AdjustedIntensityGood5AveNorm \
            "-legend=spec=Av. Good 5%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood10AveNorm \
            "-legend=spec=Av. Good 10%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood20AveNorm \
            "-legend=spec=Av. Good 20%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood100AveNorm \
            "-legend=spec=Av. Good 100%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityTopAveNorm \
            "-legend=spec=Av. Top 5%" -range=ymin=1 -mode=y=log,y=special -end \
      -column=VLD,AdjustedIntensityGoodP?? "-legend=edit=ZP" -range=ymin=1 -mode=y=log,y=special \
      "-title=Percentiles of intensity for good images adjusted for ND filters" -graph=line,vary \
      "-ylabel=ND-Adjusted Intensity" -end \
      -column=VLD,AdjustedIntensityStDevNorm "-legend=spec=Av. All" -range=ymin=1 -mode=y=log,y=special \
      "-title=Adjusted for ND filters and normalized to average value for each VLD" \
      "-ylabel=St. Dev. of Adjusted Intensity" \
      -column=VLD,AdjustedIntensityGood5StDevNorm \
            "-legend=spec=Av. Good 5%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood10StDevNorm \
            "-legend=spec=Av. Good 10%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood20StDevNorm \
            "-legend=spec=Av. Good 20%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood100StDevNorm \
            "-legend=spec=Av. Good 100%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityTopStDevNorm \
            "-legend=spec=Av. Top 5%" -range=ymin=1 -mode=y=log,y=special -end \
      -column=VLD,AdjustedIntensityQRangeNorm "-legend=spec=Av. All" -range=ymin=1 -mode=y=log,y=special \
      "-title=Adjusted for ND filters and normalized to average value for each VLD" \
      "-ylabel=Quartile Range of Adjusted Intensity" \
      -column=VLD,AdjustedIntensityGood5QRangeNorm \
            "-legend=spec=Av. Good 5%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood10QRangeNorm \
            "-legend=spec=Av. Good 10%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood20QRangeNorm \
            "-legend=spec=Av. Good 20%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood100QRangeNorm \
            "-legend=spec=Av. Good 100%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityTopQRangeNorm \
            "-legend=spec=Av. Top 5%" -range=ymin=1 -mode=y=log,y=special -end \
      -column=VLD,AdjustedIntensityDRangeNorm "-legend=spec=Av. All" -range=ymin=1 -mode=y=log,y=special \
      "-title=Adjusted for ND filters and normalized to average value for each VLD" \
      "-ylabel=Decile Range of Adjusted Intensity" \
      -column=VLD,AdjustedIntensityGood5DRangeNorm \
            "-legend=spec=Av. Good 5%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood10DRangeNorm \
            "-legend=spec=Av. Good 10%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood20DRangeNorm \
            "-legend=spec=Av. Good 20%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityGood100DRangeNorm \
            "-legend=spec=Av. Good 100%" -range=ymin=1 -mode=y=log,y=special  \
      -column=VLD,AdjustedIntensityTopDRangeNorm \
            "-legend=spec=Av. Top 5%" -range=ymin=1 -mode=y=log,y=special -end \
      -column=VLD,xSpotCenterAve "-legend=spec=Av. All" \
      "-ylabel=x Spot Center" \
      -column=VLD,xSpotCenterGood5Ave "-legend=spec=Av. Good 5%" \
      -column=VLD,xSpotCenterGood10Ave "-legend=spec=Av. Good 10%" \
      -column=VLD,xSpotCenterGood20Ave "-legend=spec=Av. Good 20%" \
      -column=VLD,xSpotCenterGood100Ave "-legend=spec=Av. Good 100%" \
      -column=VLD,xSpotCenterTopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,xSpotCenterAve "-legend=spec=Av. All" \
      "-ylabel=x Spot Centroid" \
      -column=VLD,xSpotCentroidGood5Ave "-legend=spec=Av. Good 5%" \
      -column=VLD,xSpotCentroidGood10Ave "-legend=spec=Av. Good 10%" \
      -column=VLD,xSpotCentroidGood20Ave "-legend=spec=Av. Good 20%" \
      -column=VLD,xSpotCentroidGood100Ave "-legend=spec=Av. Good 100%" \
      -column=VLD,xSpotCentroidTopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,xSpotSigmaAve "-legend=spec=Av. All" \
      "-ylabel=x Sigma" -unsup=y \
      -column=VLD,xSpotSigmaGood5Ave "-legend=spec=Av. Good 5%"  \
      -column=VLD,xSpotSigmaGood10Ave "-legend=spec=Av. Good 10%"  \
      -column=VLD,xSpotSigmaGood20Ave "-legend=spec=Av. Good 20%"  \
      -column=VLD,xSpotSigmaGood100Ave "-legend=spec=Av. Good 100%"  \
      -column=VLD,xSpotSigmaTopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,xSpotRange65Ave "-legend=spec=Av. All" \
      "-ylabel=x Range65" -unsup=y \
      -column=VLD,xSpotRange65Good5Ave "-legend=spec=Av. Good 5%" \
      -column=VLD,xSpotRange65Good10Ave "-legend=spec=Av. Good 10%" \
      -column=VLD,xSpotRange65Good20Ave "-legend=spec=Av. Good 20%" \
      -column=VLD,xSpotRange65Good100Ave "-legend=spec=Av. Good 100%" \
      -column=VLD,xSpotRange65TopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,ySpotCenterAve "-legend=spec=Av. All" \
      "-ylabel=y Spot Center" \
      -column=VLD,ySpotCenterGood5Ave "-legend=spec=Av. Good 5%" \
      -column=VLD,ySpotCenterGood10Ave "-legend=spec=Av. Good 10%" \
      -column=VLD,ySpotCenterGood20Ave "-legend=spec=Av. Good 20%" \
      -column=VLD,ySpotCenterGood100Ave "-legend=spec=Av. Good 100%" \
      -column=VLD,ySpotCenterTopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,ySpotCentroidAve "-legend=spec=Av. All" \
      "-ylabel=y Spot Centroid" \
      -column=VLD,ySpotCentroidGood5Ave "-legend=spec=Av. Good 5%" \
      -column=VLD,ySpotCentroidGood10Ave "-legend=spec=Av. Good 10%" \
      -column=VLD,ySpotCentroidGood20Ave "-legend=spec=Av. Good 20%" \
      -column=VLD,ySpotCentroidGood100Ave "-legend=spec=Av. Good 100%" \
      -column=VLD,ySpotCentroidTopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,ySpotSigmaAve "-legend=spec=Av. All" \
      "-ylabel=y Sigma" -unsup=y \
      -column=VLD,ySpotSigmaGood5Ave "-legend=spec=Av. Good 5%"  \
      -column=VLD,ySpotSigmaGood10Ave "-legend=spec=Av. Good 10%"  \
      -column=VLD,ySpotSigmaGood20Ave "-legend=spec=Av. Good 20%"  \
      -column=VLD,ySpotSigmaGood100Ave "-legend=spec=Av. Good 100%"  \
      -column=VLD,ySpotSigmaTopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,ySpotRange65Ave "-legend=spec=Av. All" \
      "-ylabel=y Range65" -unsup=y \
      -column=VLD,ySpotRange65Good5Ave "-legend=spec=Av. Good 5%" \
      -column=VLD,ySpotRange65Good10Ave "-legend=spec=Av. Good 10%" \
      -column=VLD,ySpotRange65Good20Ave "-legend=spec=Av. Good 20%" \
      -column=VLD,ySpotRange65Good100Ave "-legend=spec=Av. Good 100%" \
      -column=VLD,ySpotRange65TopAve "-legend=spec=Av. Top 5%" -end \
      -column=VLD,BackgroundLevelAve "-legend=spec=Av. All" \
      -column=VLD,BackgroundLevelTopAve "-legend=spec=Av. Top 5%" -end &
    cd $oldDir
}

proc SaveVLDConfig {} {
    global numberOfFiles listOfRootnames NDFilter experimentName NDFilterWheel
    global ROI spotROISize sizeLines VLDGroupList blankOut workDirectory
    global lonerRemoval backgroundHalfWidth symmetricBackgroundRemoval
    global saturationLevel centroidTolerance allowedSaturatedPixels filterWheelFile
    global cameraSensitivity
    foreach group $VLDGroupList {
        if !$numberOfFiles($group) {
            setStatus "No files chosen for group $group"
            continue
        }
        set outputData(ColumnNames) Rootname
        set outputData(ParameterNames) "NDFilterSetting ROIx0 ROIx1 ROIy0 ROIy1 blankOutx0 blankOutx1 blankOuty0 blankOuty1 spotROISizex spotROISizey sizeLinesx sizeLinesy lonerRemoval backgroundHalfWidth symmetricBackgroundRemoval saturationLevel centroidTolerance allowedSaturatedPixels NDFilterWheel1 NDFilterWheel2 NDFilterWheel3 filterWheelFile cameraSensitivity"
        set outputData(Column.Rootname) [list $listOfRootnames($group)]
        set outputData(Parameter.NDFilterSetting) [list $NDFilter($group)]
        set outputData(Parameter.ROIx0) [list $ROI(x0,$group)]
        set outputData(Parameter.ROIx1) [list $ROI(x1,$group)]
        set outputData(Parameter.ROIy0) [list $ROI(y0,$group)]
        set outputData(Parameter.ROIy1) [list $ROI(y1,$group)]
        foreach i {1 2 3} {
            set outputData(Parameter.NDFilterWheel$i) [list $NDFilterWheel($i,$group)]
        }
        set outputData(Parameter.blankOutx0) [list $blankOut(x0,$group)]
        set outputData(Parameter.blankOutx1) [list $blankOut(x1,$group)]
        set outputData(Parameter.blankOuty0) [list $blankOut(y0,$group)]
        set outputData(Parameter.blankOuty1) [list $blankOut(y1,$group)]
        set outputData(Parameter.sizeLinesx) [list $sizeLines(x,$group)]
        set outputData(Parameter.sizeLinesy) [list $sizeLines(y,$group)]
        set outputData(Parameter.spotROISizex) [list $spotROISize(x,$group)]
        set outputData(Parameter.spotROISizey) [list $spotROISize(y,$group)]
        set outputData(Parameter.lonerRemoval) [list $lonerRemoval($group)]
        set outputData(Parameter.symmetricBackgroundRemoval) [list $symmetricBackgroundRemoval($group)]
        set outputData(Parameter.backgroundHalfWidth) [list $backgroundHalfWidth($group)]
        set outputData(Parameter.saturationLevel) [list $saturationLevel($group)]
        set outputData(Parameter.centroidTolerance) [list $centroidTolerance($group)]
        set outputData(Parameter.allowedSaturatedPixels) [list $allowedSaturatedPixels($group)]
        set outputData(Parameter.filterWheelFile) [list $filterWheelFile($group)]
        set outputData(Parameter.cameraSensitivity) [list $cameraSensitivity($group)]
        if [catch {sdds save $workDirectory/$experimentName.VLD$group.settings outputData} result] {
            return -code error "$result"
        }
    }
    setStatus "Configuration saved"
}

proc ResetVLDConfig {args} {
    set confirm 0
    APSStrictParseArguments {confirm}
    if $confirm {
        set answer [APSMultipleChoice [APSUniqueName .] -question \
                      "Really?  Reset configuration (clears all file lists)." \
                      -labelList "Yes No" -returnList "1 0"]
        if $answer==0 {
            return
        }
    }
    global numberOfFiles listOfRootnames NDFilter experimentName NDFilterWheel
    global ROI spotROISize sizeLines VLDGroupList blankOut workDirectory
    global lonerRemoval backgroundHalfWidth symmetricBackgroundRemoval
    global saturationLevel centroidTolerance allowedSaturatedPixels filterWheelFile
    global cameraSensitivity
    foreach group $VLDGroupList {
        set listOfRootnames($group) ""
        set numberOfFiles($group) 0
        set NDFilter($group) 0
        set NDFilter($group) 0
        set ROI(x0,$group) 50
        set ROI(x1,$group) 718
        set ROI(y0,$group) 0
        set ROI(y1,$group) 479
        set blankOut(x0,$group) -1
        set blankOut(x1,$group) -1
        set blankOut(y0,$group) -1
        set blankOut(y1,$group) -1
        foreach i {1 2 3} {
            set NDFilterWheel($i,$group) -1
        }
        set filterWheelFile($group) /home/helios/oagData/linac/FELData/NDFilters530nm.sdds
        set spotROISize(x,$group) 150
        set spotROISize(y,$group) 150
        set sizeLines(x,$group) 3
        set sizeLines(y,$group) 3
        set lonerRemoval($group) 0
        set symmetricBackgroundRemoval($group) 0
        set backgroundHalfWidth($group) 2
        set saturationLevel($group) 254
        set centroidTolerance($group) 10
        set allowedSaturatedPixels($group) 4
        set cameraSensitivity($group) 1
    }
}

proc LoadVLDConfig {} {
    global numberOfFiles listOfRootnames NDFilter experimentName NDFilterWheel
    global ROI spotROISize sizeLines VLDGroupList blankOut workDirectory
    global lonerRemoval backgroundHalfWidth symmetricBackgroundRemoval
    global saturationLevel centroidTolerance allowedSaturatedPixels filterWheelFile
    global cameraSensitivity
    ResetVLDConfig
    foreach group $VLDGroupList {
        if ![file exists $workDirectory/$experimentName.VLD$group.settings] {
            setStatus "No configuration for VLD $group."
            continue
        }
        if [catch {sdds load $workDirectory/$experimentName.VLD$group.settings inputData} result] {
            return -code error "$result"
        }
        set listOfRootnames($group) [lindex $inputData(Column.Rootname) 0]
        set numberOfFiles($group) [llength $listOfRootnames($group)]
        set NDFilter($group) [lindex $inputData(Parameter.NDFilterSetting) 0]
        foreach type {x0 x1 y0 y1} {
            set ROI($type,$group) [lindex $inputData(Parameter.ROI$type) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) blankOutx0]!=-1 {
            foreach type {x0 x1 y0 y1} {
                set blankOut($type,$group) [lindex $inputData(Parameter.blankOut$type) 0]
            }
        }
        foreach type {x y} {
            set spotROISize($type,$group) [lindex $inputData(Parameter.spotROISize$type) 0]
            set sizeLines($type,$group) [lindex $inputData(Parameter.sizeLines$type) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) cameraSensitivity]!=-1 {
            set cameraSensitivity($group) [lindex $inputData(Parameter.cameraSensitivity) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) backgroundHalfWidth]!=-1 {
            set backgroundHalfWidth($group) [lindex $inputData(Parameter.backgroundHalfWidth) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) lonerRemoval]!=-1 {
            set lonerRemoval($group) [lindex $inputData(Parameter.lonerRemoval) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) symmetricBackgroundRemoval]!=-1 {
            set symmetricBackgroundRemoval($group) [lindex $inputData(Parameter.symmetricBackgroundRemoval) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) saturationLevel]!=-1 {
            set saturationLevel($group) [lindex $inputData(Parameter.saturationLevel) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) centroidTolerance]!=-1 {
            set centroidTolerance($group) [lindex $inputData(Parameter.centroidTolerance) 0]
        }
        if [lsearch -exact $inputData(ParameterNames) allowedSaturatedPixels]!=-1 {
            set allowedSaturatedPixels($group) [lindex $inputData(Parameter.allowedSaturatedPixels) 0]
        }
        foreach i {1 2 3} {
            if [lsearch -exact $inputData(ParameterNames) NDFilterWheel$i]!=-1 {
                set NDFilterWheel($i,$group) [lindex $inputData(Parameter.NDFilterWheel$i) 0]
            }
        }
        if [lsearch -exact $inputData(ParameterNames) filterWheelFile]!=-1 {
            set filterWheelFile($group) [lindex $inputData(Parameter.filterWheelFile) 0]
        }
    }
    setStatus "Configurations loaded"
}

proc CheckDQSJob {args} {
    set directory ""
    set jobID ""
    set group ""
    set experimentName ""
    APSStrictParseArguments {directory jobID group experimentName}
    set fileList [glob -nocomplain $directory/V${group}$experimentName.o$jobID.*]
    if [llength $fileList]==0 {
        setStatus "Warning: DQS job $directory/V${group}$experimentName.o$jobID not found"
        setStatus "Will check again in 30 seconds."
        after 30000 "CheckDQSJob -directory $directory -jobID $jobID -group $group -experimentName $experimentName" 
    }
}

set cameraSensitivityFile /home/helios/oagData/linac/FELData/cameraSensitivity.sdds
proc ReadCameraSensitivityData {} {
    global cameraSensitivityFile VLDGroupList cameraSensitivity
    set newFile [APSFileSelectDialog [APSUniqueName .] \
                   -title "Camera Sensitivity File Selection" \
                   -path [file dirname $cameraSensitivityFile] \
                   -pattern cameraSensitivity-*.sdds \
                   -contextHelp "Select a file to use supplying camera sensitivities"]
    if [string length $newFile] {
        set cameraSensitivityFile $newFile
    } 
    if ![string length cameraSensitivityFile] {
        return
    }
    if [catch {sdds load $cameraSensitivityFile camSens} result] {
        return -code error "$camSens"
    }
    foreach column {Station Sensitivity} {
        if [lsearch -exact $camSens(ColumnNames) $column]==-1 {
            return -code error "Column $column not found in $newFile"
        }
        set ${column}List [lindex $camSens(Column.$column) 0]
    }
    foreach group $VLDGroupList {
        set station [os editstring 1f1d $group]
        if [set index [lsearch -exact $StationList $station]]==-1 {
            setStatus "Warning: no camera sensitivity data for station $station"
            set cameraSensitivity($group) 1
        } else {
            set cameraSensitivity($group) [lindex $SensitivityList $index]
        }
    }
}

if [string compare $env(USER) borland]==0 {
    set experimentName dataset4
    set workDirectory /oagscratch/OAGSCRATCH/borland/FELData/0009/30
    set dataDirectory /oagscratch/OAGSCRATCH/borland/FELData/0009/30
} else {
    set experimentName dataset
    set workDirectory [pwd]
    set dataDirectory [pwd]
}
set plotWithColor 0

BuildApplication

