# $Log: not supported by cvs2svn $
# Revision 1.27  2012/02/14 19:37:11  shang
# modified to obtain the scan together gaps from config file instead of hard-coded.
#
# Revision 1.26  2012/02/01 16:12:03  shang
# modified to check only the gaps that were requested to allow other gaps stay as it is for single gap scan.
#
# Revision 1.25  2011/12/22 20:02:49  shang
# made A,B,C,D blades to select/de-select together for removing bad blades.
#
# Revision 1.24  2011/10/31 20:39:33  shang
# corrected the message for loading offsets and rates.
#
# Revision 1.23  2011/10/31 20:17:35  shang
# added loadOffset and loadRate option to APSMpSRGapBriefScan
#
# Revision 1.22  2011/10/31 18:31:40  shang
# added "return $moveList" to APSSRMoveIDGaps to return the moving ID gpas.
#
# Revision 1.21  2011/09/29 23:13:24  shang
# replaced "-sectorList $sectorList" by "-sectorCount 35" in APSSRSectorButtons statement due to library changes.
#
# Revision 1.20  2011/08/16 13:33:18  shang
# added the missing "\" bach for sddsexpfit command.
#
# Revision 1.19  2011/08/16 13:31:12  shang
# put the initial guess value back for sddsexpfit because it could not fit ID12 after removing it, need find a better initial values for 16IDus.
#
# Revision 1.18  2011/08/16 13:05:50  shang
# removed the initial guess of sddsexpfit since it does not fit ID16us well.
#
# Revision 1.17  2010/10/01 22:22:42  shang
# fixed the problem in loading rates and offsets when the input directory names has "/" character in the end, the short file names in that directory were not correctly obtained.
#
# Revision 1.16  2010/01/29 22:18:50  soliday
# Updated the callback procedure related to selected the SR sector buttons.
#
# Revision 1.15  2009/11/09 20:33:48  shang
# modified the gap brief scan procedure so that it excludes the fixed gap into scan list.
#
# Revision 1.14  2009/01/26 19:23:14  shang
# added 35 ID to the bad blade selection widget and fixed the error that complained no such variable of badBladesS35ID:P1:A.
#
# Revision 1.13  2008/09/25 16:37:39  shang
# modified to stagger closing the gaps.
#
# Revision 1.12  2008/07/03 14:50:30  shang
# now uses APSCheckAndWaitForIDs for checking and waiting ID gaps move to the setpoints, and moved the checking fixed gaps, ID14 gaps into APSSRCheckUndulatorScanType procedure.
#
# Revision 1.11  2008/06/17 21:49:22  shang
# changed to wait for all the gaps stopping moving in brief scan.
#
# Revision 1.10  2008/06/17 17:28:43  shang
# fixed a typo, replaced "reutn" by "return"
#
# Revision 1.9  2008/03/11 17:07:34  shang
# added option of scan dual gaps together or with one gap fixed or scan independently, tested with Glenn Decker.
#
# Revision 1.8  2007/11/13 19:51:44  shang
# simplified the full gap scan pem for operator use.
#
# Revision 1.7  2007/04/03 15:18:27  shang
# per Glenn's requst, removed the review plots for installing FF and loading offset.
#
# Revision 1.6  2007/03/27 18:33:44  shang
# added lockedIDList to APSSRMoveIDGaps to not move the locked IDs which can not be moved.
#
# Revision 1.5  2007/03/14 02:26:54  emery
# Add escaped quotes for a list variable in callback command.
#
# Revision 1.4  2007/03/07 15:01:05  shang
# commented out APSSRUpdateGapMonFile since there seems to be problem.
#
# Revision 1.3  2007/02/07 06:14:31  emery
# Indented code.
# Correct $rateFile file name for checking the rows in APSSRGapBriefScanLoad.
#
# Revision 1.2  2006/12/20 18:20:21  shang
# install after fully testing.
#
# Revision 1.1  2006/11/28 17:46:36  shang
# first version.
#

proc APSSRGapScanReturnProc {} {
    update
    bell
    APSSRGapScanCallbackProc
}

proc APSSRGapScanCallbackProc {args} {
    set ok 0
    set outputFile ""
    set IDtagList ""
    APSStrictParseArguments {ok outputFile IDtagList}
    
    if {!$ok && [string length $outputFile]} {
        file delete $outputFile
        return
    }
    if ![file exists $outputFile] {
        APSAlertBox .error -errorMessage "APSCallbackProc: $outputFile does not exist yet -- the problem may come from sddsexperiment which was supposed to create this output file, please check the messages in exec log window!"
        return
    }
    
    if [catch {exec sddsprocess -nowarning $outputFile $outputFile.1 \
                   "-reprint=parameter,Description,IDs: [APSMakeSafeQualifierString [join $IDtagList ,]]"  } result] {
        return -code error "$result"
    }
    exec mv $outputFile.1 $outputFile
    file delete -force ${outputFile}~
}

proc APSSRGetIDGapBriefInfo {args} {
    global AllIDList CUSectorList IDUseFlag APSMpSRGapBriefScan
    
    if ![info exist APSMpSRGapBriefScan(pause)] {
        set APSMpSRGapBriefScan(pause) 10
    }
    if ![info exist APSMpSRGapBriefScan(numToAve)] {
        set APSMpSRGapBriefScan(numToAve) 2
    }
    
    set  CUdataDir /home/helios/oagData/feedForwardFiles/IDgap
    if [catch {exec sddsprocess $CUdataDir/cantingLogic.sdds -pipe=out \
                   -filter=col,TreatedAsCU,1,1 \
                   | sdds2stream -pipe -col=Sector} CUSectorList] {
        return -code error "APSSRGetIDGapInfo: $CUSectorlist"
    }
    if [catch {exec sdds2stream -col=DeviceName /home/helios/oagData/sr/IDs/IDdevices.sdds} AllIDList] {
        return -code error $AllIDList
    }
    if [catch {exec sdds2stream -col=Default /home/helios/oagData/sr/IDs/IDdevices.sdds} defaultList] {
        set defaultList ""
    }
    if [string length $defaultList] {
        foreach ID $AllIDList default $defaultList {
            set IDUseFlag($ID) $default
        }
    } else {
        foreach ID $AllIDList default $defaultList {
            set IDUseFlag($ID) 0
        }
    }
    APSUpdateIDScanSelection ID03ds
    APSUpdateIDScanSelection ID26ds
    APSUpdateIDScanSelection ID30ds
}

proc APSMakeIDGapBriefScanSelection {args} {
    global IDUseFlag AllIDList APSMpSRGapBriefScan CUSectorList
    set type brief
    APSStrictParseArguments {type}
    
    if [catch {APSSRGetIDGapBriefInfo} result] {
        return -code error $result
    }
    set APSMpSRGapBriefScan(closeInterval) 0.1
    foreach id $AllIDList {
        lappend variableList IDUseFlag($id)
        lappend commandList "APSUpdateIDScanSelection $id"
    }

    APSDialogBox .idscan -name "ID Gap Scan Selection" -cancelButton 0
    set parent .idscan.userFrame
    APSCheckButtonFrame .id -parent .idscan.userFrame -buttonList $AllIDList -variableList $variableList -allNone 1 \
        -limitPerRow 5 \
        -label "Select IDs" -commandList $commandList \
        -contextHelp "Check ID gaps to include in the scan. Use /home/helios/oagData/sr/gapScans/feedforwardInputFiles/gapValuesBrief.sdds file for gap scan values, i.e., gap scans from 20mm to 35mm as shown in the file"
    
    APSLabeledEntry .pause -parent .idscan.userFrame -label "Pause after gap change (s):" -width 20 \
        -textVariable APSMpSRGapBriefScan(pause) -contextHelp "Length of pause after gap change is completed to allow orbit correction to converge."
    APSLabeledEntry .ave -parent .idscan.userFrame -label "Readings to average:" -width 20 \
            -textVariable APSMpSRGapBriefScan(numToAve) -contextHelp "Number of readings to average at each gap value."
    
    APSLabeledEntry .interval -parent .idscan.userFrame -label "Interval of closing gaps (seconds):" -width 20 \
      -textVariable APSMpSRGapBriefScan(closeInterval) \
      -contextHelp "Due to on-going restrictions on the SR rf systems, we need to tagger the closing of the gaps by a selectable time interval, i.e. issuing the next gap close command after T seconds of the previous one."
    
    tkwait window .idscan
    
    set selectList ""
    foreach id $AllIDList {
        if $IDUseFlag($id) {
            lappend selectList $id
        }
    }
    return $selectList
}



proc APSSRGapBriefScan {args} {
    set IDList ""
    set outputDir /home/helios/oagData/sr/gapScans/data/gapScanBrief
    set rootname IDscan
    set statusCallback "APSSetVarAndUpdate status"
    set pause 60
    set preview 0
    set transferOrbit 0 
    set readingType msAve
    set numToAve 10
    set fixedGapList ""
    set fixedIDList ""
    set togetherList ""
    set ID01OffsetList ""
    set keepFFrunning 0
    APSStrictParseArguments {IDList outputDir rootname statusCallback keepFFrunning \
                               pause preview  readingType numToAve fixedGapList \
                               fixedIDList togetherList ID01OffsetList}
    global APSMpSRGapBriefScan
   
    if ![llength $IDList] {
        return -code error "No IDs selected for brief gap scan."
    }
    if ![file exist $outputDir] {
        return -code error "Error: directory $outputDir does not exist!"
    }
    if ![file isdirectory $outputDir] {
        return -code error "$outputDir (in the Output dir entry) is not a directory."
    }
    set tmpstring [APSTmpString]
    if [catch {open $outputDir/$tmpstring "w"} fd] {
        close $fd
        return -code error "You do not have permission to write in $outputDir directory"
    } else {
        close $fd
        file delete -force $outputDir/$tmpstring
    }
    if [llength $fixedIDList] {
        foreach ID $fixedIDList {
            set index [lsearch $IDList $ID]
            if $index>=0 {
                set IDList [lreplace $IDList $index $index]
            }
        }
    }
    set expFileList ""
    set tmpRoot /tmp/[APSTmpString]
    set IDgapValues $tmpRoot.gapValues
    APSAddToTempFileList $IDgapValues
    set inputFileDir /home/helios/oagData/sr/gapScans/feedforwardInputFiles
    set gapScanValuesFile $inputFileDir/gapValuesBrief.sdds
    set IDtagList ""
    foreach item $IDList {
        scan $item ID%s tag
        lappend IDtagList $tag
        if !$preview {
            set expFile $tmpRoot.$item.exp
            if [catch {exec replaceText $inputFileDir/IDscan.template1.exp $expFile \
                           -original=<ID>,<values_file> \
                           -replace=$item,$IDgapValues} result] {
                return -code error $result
            }
            lappend expFileList $expFile
        }
        lappend IDgapList ${item}gap
    }
    
    if [catch {exec sddsconvert $gapScanValuesFile $IDgapValues \
                   -retain=col,Step,[join $IDgapList ,] } result] {
        return -code error $result
    }
    if $preview {
        exec sddsplot $IDgapValues -column=Step,ID* -separate -graph=symbol &
        return
    }
    if [catch {APSSRCheckXBPMGains -IDList $IDList} result] {
        return -code error $result
    }
    if [catch {APSSRCheckGapScanConditions -keepFFrunning $keepFFrunning} result] {
	return -code error $result
    }
#    APSSRUpdateGapScanMonitorFiles
    set IDgapTests /tmp/[APSTmpString].gapTests
    if [catch {exec sddsmakedataset -col=GapName,type=string -data=[join $IDList ,] -pipe=out \
                 | sddsprocess -pipe=in $IDgapTests -edit=col,ControlName,GapName,ei/:GlobalBusy.VAL/ } result] {
        return -code error "APSSRGapFullScan: Error generating tests file: $result"
    }
    APSAddToTempFileList -ID fitscan -fileList $IDgapTests
    set expFile /tmp/[APSTmpString].exp 
    if $transferOrbit {
        set template $inputFileDir/IDFitscan.template2.exp
    } else {
        set template $inputFileDir/IDFitscan.template3.exp
    }
    if [catch {exec replaceText $template $expFile \
                   -original=<IDlist>,<pause>,<numToAve>,<readingType>,<testFile> \
                   "-replace=$IDtagList,$pause,$numToAve,$readingType,$IDgapTests"} result] {
        return -code error $result
    }
    lappend expFileList $expFile
    
    set expFile /tmp/[APSTmpString].exp
    APSAddToTempFileList $expFile
    if [catch {eval exec cat $expFileList > $expFile} result] {
        return -code error $result
    }
    
    catch {eval file delete $expFileList} 
    set snapshot [lindex [APSSCRMakeSnapshotFilename -machine SR] 0]
    regsub {SR} $snapshot ${rootname}- snapshot0 
    set outputFile $outputDir/${snapshot0}.sdds
    global briefScan
    set briefScan 0
    APSExecLog [APSUniqueName .] -name "ID scan" -width 100 \
        -unixCommand "sddsexperiment $expFile $outputFile -verbose" \
        -callback "APSSRGapScanCallbackProc -ok 1 -outputFile $outputFile -IDtagList \"$IDtagList\"; set briefScan done" \
        -cancelCallback "APSSRGapScanCallbackProc -ok 0 -outputFile $outputFile -IDtagList \"$IDtagList\"; set briefScan canceled" \
        -abortCallback "APSSRGapScanCallbackProc -ok 0 -outputFile $outputFile -IDtagList \"$IDtagList\"; set briefScan aborted"
    tkwait variable briefScan
    if [string compare $briefScan "done"]!=0 {
        return -code error "APSSRGapBriefScan was $briefScan"
    }
    if [catch {exec sddsprocess $outputFile $outputFile.1 "-print=par,FixedGaps,[join $fixedIDList]" \
                   "-print=par,FixedGapValues,[join $fixedGapList]" "-print=par,TogetherSectors,[join $togetherList]" } result] {
        return -coded error "APSSRGapFullScan(1 -- sddsprocess): $result"
    }
    exec mv $outputFile.1 $outputFile
   # puts $ID01OffsetList
    if [llength $ID01OffsetList] {
        set ID01BladeList {ID01A1:Corrected ID01B1:Corrected ID01C1:Corrected ID01D1:Corrected ID01A2:Corrected ID01B2:Corrected ID01C2:Corrected ID01D2:Corrected}
        if [catch {exec sddsmakedataset -col=ID01BladeName,type=string -data=[join $ID01BladeList ,] \
                     -col=BladeOffset,type=double -data=[join $ID01OffsetList ,] $outputFile.ID01dsOffset } result] {
            return -code error "Error creating $outputFile.ID01dsOffset : $result"
        }
    }
    return $outputFile
}

proc APSSRCheckXBPMGains {args} {
    set IDList ""
    set statusCallback ""
    APSStrictParseArguments {IDList statusCallback}

    #xbpm gain not available for APSU
    return
    global CUSectorList
    
    if [string length $statusCallback] {
        eval $statusCallback {"Checking Xbpm gains..." }
    }
    set tmpRoot /tmp/[APSTmpString]
    
    APSAddToTmpFileList -ID ganscan -fileList "$tmpRoot.sdds"
    set pvList ""
    foreach id [lsort $IDList] {
        set sector [scan $id ID%ld]
        if [lsearch $CUSectorList $sector]>=0 {
            #for CU sector, us gap is associated with P1, ds gap is associated with P2
            if [regexp {us} $id] {
                lappend pvList S${sector}ID:P1:ms:x:GainAO
                lappend pvList S${sector}ID:P1:ms:y:GainAO
            } else {
                lappend pvList S${sector}ID:P2:ms:x:GainAO
                lappend pvList S${sector}ID:P2:ms:y:GainAO
            }
        } else {
            lappend pvList S${sector}ID:P1:ms:x:GainAO
            lappend pvList S${sector}ID:P1:ms:y:GainAO
            lappend pvList S${sector}ID:P2:ms:x:GainAO
            lappend pvList S${sector}ID:P2:ms:y:GainAO
        }
    }
    
    if [catch {exec sddsmakedataset -pipe=out  \
                   -col=ControlName,type=string -data=[join $pvList ,] \
                   | sddscasr -save -pipe \
                   | sddsprocess -pipe -scan=col,Value,ValueString,%lf \
                   | sddsprocess -pipe=in $tmpRoot.sdds \
                   -nowarnings -filter=col,Value,0,0} result] {
        return -code error $result
    }
    
    set rows [exec sdds2stream $tmpRoot.sdds -rows=bare]
    if $rows {
        set pvs [exec sdds2stream $tmpRoot.sdds -col=ControlName]
        APSAlertBox .warning -beep once -errorMessage "Following xbpm gains are zero:\n$pvs"
        return -code error "The xbpms gains of [join $pvs ,] are zero."
    }
}

proc APSSRUpdateGapScanMonitorFiles {args} {
    set statusCallback ""
    APSStrictParseArguments {statusCallback}
    set inputFileDir /home/helios/oagData/sr/gapScans/feedforwardInputFiles
    set time0 [file mtime $inputFileDir/IDcorrectedPVs.mon]
    set time1 [file mtime /home/helios/oagData/sr/BPMStatus/config.sdds]
    set time2 [file mtime /usr/local/iocapps/iocinfo/pvdata/all/iocRecNames.sdds]
    if {$time0<$time1 || $time0<$time2} {
        set oldDir [pwd]
        cd $inputFileDir
        if [string length $statusCallback] {
            eval $statusCallback {"updating monitor files..."}
        }
        catch {exec makeGapScanMonitorFile} result
        if [string length $statusCallback] {
            eval $statusCallback {"done."}
        }
        cd $oldDir
    }
}

# "FitData proc does curve fitting only and no generation of corrected"
# "blade or position signals."
# "Program to fit gap scan data with an exponential function"
# "usage: <sector> inputFile outputFile"
# "where <sector> is a two digit number plus optional"
# "ds / us designation. Thus, e.g.,  07 and not just 7,"
# "and for straights with two devices, "
# "sector = 02ds, 02us, 23us, 23ds etc."
# 'The outfile is optional, default name is ID${sector}.proc'
# "It _will_ overwrite the output file it already exists,"
# "written by G. Decker, 040909"

proc APSSRGapBriefScanOneGapFit {args} {
    set sector ""
    set inputFile ""
    set outputFile ""
    set fitStart 0.1
    set fitFactor 340
    set fitRate -0.2
    set CUSectorList ""
    APSStrictParseArguments {sector inputFile outputFile fitStart fitFactor fitRate CUSectorList}

    if ![string length $sector] {
        return -code error "ID sector is not given"
    }
    if {![string length $inputFile] || ![file exists $inputFile]} {
        return -code error "inputFile is not given or does not exist!"
    }
    if ![string length $outputFile] {
        set outputFile ID${sector}.proc
    }
    # usds = US or DS, to sort out the inconsistent ID naming conventions
    # Default insertion device gap pv name has neither US or DS contained
    # in it.
    set sector1 [scan $sector %ld]
    
    if [regexp {us} $sector a b] {
        set usds US
    } elseif [regexp {ds} $sector a b] {
        set usds DS
    } else {
        set usds ""
    }
    set gapLimit  10
    
    # After filtering out data with gap less tha gapLimit, grab corrected
    # blade signals and gap for the sector of interest from infile and
    # place it in outfile
    #not should change 180 back to 40 in real exp (here use 185, is just for testing purpose)
    set sector2 [format %02ld $sector1]
    if [catch {exec sddsprocess $inputFile -pipe=out \
                   -filter=col,ID${sector2}${usds}AverageGap,${gapLimit},40 \
                   | sddsconvert -pipe=in $outputFile -del=par,* \
                   -retain=col,ID${sector2}*Corrected,ID${sector2}${usds}AverageGap } result] {
        return -code error "Error in processing $inputFile :$result\nPlease check if the average is between 19 and 40 mm"
    }
    # P1 uses ABCD configuration
    # Figure out if P2 uses ABCD or ABEF configuration, etc.
    # default is ABEF
    # P2ABCD = 0 means ABEF,
    # P2ABCD = 1 means ABCD
    # P2ABCD = 2 means upstream canted undulator
    # P2ABCD = 3 means downstream canted undulator
    
    if {[lsearch $CUSectorList $sector1]>=0} {
        set prefix ID[format %02ld $sector1]
        switch $usds {
            US {
                # The upstream canted undulator is associated only with P1:
                # set bladeList {A1 B1 C1 D1}
                set bladeList [list ${prefix}A1:Corrected ${prefix}B1:Corrected \
                                   ${prefix}C1:Corrected ${prefix}D1:Corrected]
            }
            DS {
                # The downstream  canted undulator is associated only with P2:
                # set bladeList {A2 B2 C2 D2}
                set bladeList [list ${prefix}A2:Corrected ${prefix}B2:Corrected \
                                   ${prefix}C2:Corrected ${prefix}D2:Corrected]
            }
            default {
                return -code error "Canted undulator sector $sector1 is not associated either with upstream or downstream"
            }
        }
    } else {
        #all other sector
        catch {exec sddsquery -col $outputFile | grep :Corrected} bladeList
    }
    # Fitting is only performed for gap values > gapLimit, because
    # photoelectron space charge effects cause the curves
    # to deviate from exponential behaviour at small gaps.
    # This procedure does the best job of extracting the
    # baseline offset value, and ultimately straightens out the
    # position vs. gap variation at large gap.
    
    # Get background level, rate, and factor from expfit of
    # corrected (i.e. normalized to beam current) blade data.
    # Add fitted data, residuals, and fit coefficients to outfile.
    set tmpfile /tmp/[APSTmpString].$sector
    global BriefGapScanPlotoption
    #the offset was read from blade voltage when the ID01 gaps are open
    if {$sector=="01ds"} {
        if ![file exist $inputFile.ID01dsOffset] {
            return -code error "Error: ID01 offset file $inputFile.ID01dsOffset does not exist, can not process data for ID01ds"
        }
        set ID01OffsetList [exec sdds2stream -col=BladeOffset $inputFile.ID01dsOffset]
        set ID01BladeList [exec sdds2stream -col=ID01BladeName $inputFile.ID01dsOffset]
    }
    foreach blade $bladeList {
        if {$sector=="01ds"} {
            set index [lsearch $ID01BladeList $blade]
            set offset [lindex $ID01OffsetList $index]
            set extraOpt -fixValue=constant=$offset
        } else {
            set extraOpt ""
        }
        APSAddToTmpFileList -ID 1 -fileList $tmpfile.$blade
        if [catch {eval exec sddsexpfit -full $outputFile $tmpfile.$blade -tolerance=1.0e-14 \
                       -col=ID${sector2}${usds}AverageGap,$blade $extraOpt
                   #    -startValue=constant=$fitStart,factor=$fitFactor,rate=$fitRate
            exec sddsxref $outputFile $tmpfile.$blade  -pipe=out -nowarnings \
                       -transfer=par,expfitConstant,expfitRate,expfitFactor \
                       -take=${blade}Fit,${blade}Residual \
                       | sddsprocess -pipe -process=${blade}Residual,largest,expfitResidual \
                       | sddsconvert -pipe=in $outputFile -nowarning \
                       -rename=par,expfitConstant=${blade}Const \
                       -rename=par,expfitRate=${blade}Rate \
                       -rename=par,expfitFactor=${blade}Factor \
                       -rename=par,expfitResidual=${blade}FitError } result] {
            return -code error $result
        }
    }
}

proc APSSRGapBriefScanDataFit {args} {
    set inputFile ""
    set statusCallback "APSSetVarAndUpdate status"
    set tolerance 1
    set fitStart 0.1
    set fitFactor 340
    set fitRate -0.2
    set CUSectorList ""
    APSStrictParseArguments {inputFile statusCallback tolerance fitStart fitFactor fitRate CUSectorList}
    
    
    if ![file exist $inputFile] {
        return -code error "$outputFile doe not exist!"
    }
    set parNames [exec sddsquery -par $inputFile]
    set fixedIDList ""
    set fixedGapList ""
    set togetherList {3 26 30}
    if [lsearch $parNames FixedGaps]>=0 {
        set fixedIDList [join [exec sdds2stream -par=FixedGaps $inputFile]]
        set fixedGapList [join [exec sdds2stream -par=FixedGapValues $inputFile]]
        set togetherList [join [exec sdds2stream -par=TogetherSectors $inputFile]]
    }
    set IDList ""
    if [catch {exec sddsquery -col $inputFile | grep GapSP} gaps] {
        return -code error $gaps
    } 
    if ![string length $gaps] {
        return -code error "no IDs varied in $inputFile" 
    }
    foreach gap [lsort $gaps] { 
       # set id [os editstring %/ID//%/GapSP// $gap]
        set id [regsub "GapSP" [regsub {ID} $gap ""] ""]
        lappend IDList $id
    }
    set fileList ""
    set tmpfile /tmp/[APSTmpString]
    foreach sector $IDList {
        set sector1 [scan $sector %ld]
        if [lsearch $togetherList $sector1]>=0 {
            #for these three sectors, the us and ds are scanned together, so only need process one gap
            if [regexp {us} $sector] {
                set dsgap ID[format %02d $sector1]ds
                if [lsearch $IDList $dsgap]>=0 {
                    continue
                }
            }
        }
        if [string length $statusCallback] {
            eval $statusCallback {"Processing sector $sector ..."}
        }
        if [catch {APSSRGapBriefScanOneGapFit -sector $sector -inputFile $inputFile \
                     -fitStart $fitStart -fitFactor $fitFactor -fitRate $fitRate \
                     -CUSectorList $CUSectorList \
                     -outputFile $tmpfile.ID${sector}.sdds } result] {
            return -code error "Unable processing data for ID$sector: $result"
        }
        lappend fileList $tmpfile.ID${sector}.sdds
    }
    APSAddToTmpFileList -ID fitgapscan -fileList $fileList
    set procfile ${inputFile}.proc
    set offsetfile ${inputFile}.offset
    set ratefile ${inputFile}.rate
    if [file exist $inputFile.fit] {
        catch {file delete -force $inputFile.fit}
    }
    if {[llength $fileList]>1} {
        if [catch {eval exec sddsxref $fileList -take=* -transfer=par,* ${inputFile}.fit } result] {
            return -code error $result
        }
    } elseif {[llength $fileList]==1} {
        exec cp $fileList ${inputFile}.fit
    } else {
        if [string length $statusCallback] {
            eval $statusCallback {"No valid data for scanned IDs."}
        }
        return -code error "APSSRGapBriefScanDataFit: No valid data for scanned IDs."
    }
   
    APSSRReviewBriefScanFitData -fitData ${inputFile}.fit -CUSectorList $CUSectorList
    #  puts stderr $fileList
 
    if [lsearch $IDList 01ds]>=0 {
        set editOpt "-reedit=col,RatePV,%/S1ID:/S1IDds:/ -reedit=col,OffsetPV,%/S1ID:/S1IDds:/"
    } else {
        set editOpt "-reedit=col,RatePV,%/S1ID:/S1IDus:/ -reedit=col,OffsetPV,%/S1ID:/S1IDus:/"
    } 
    if [catch {eval exec sddscollapse ${inputFile}.fit -pipe=out \
                 | sddscollect -pipe -collect=suffix=Rate -collect=suffix=FitError \
                 -collect=suffix=Factor -collect=suffix=Const \
                 | sddsprocess -pipe \
                 -scan=col,sector,Rootname,ID%ld,type=long \
                 -edit=col,blade,Rootname,4d%/:Corrected// \
                 -edit=col,blade1,blade,1f1d -edit=col,bpm,blade,1d \
                 -print=col,BladeName,S%ldID:P%s:%s,sector,bpm,blade1 \
                 -print=col,OffsetPV,S%ldID:P%s:%s:OffsetP,sector,bpm,blade1 \
                 -edit=col,RatePV,OffsetPV,%/OffsetP/RateP/ \
                 | sddsprocess -pipe=in $procfile $editOpt } result] {
        return -code error "APSSRGapBriefScanDataFit: $result"
    }
    
    if [catch {exec sddsprocess $procfile -pipe=out \
                   -print=col,ValueString,%.12f,Const \
                   | sddsconvert -pipe=in $offsetfile -retain=col,ValueString,OffsetPV,FitError,BladeName \
                   -rename=col,OffsetPV=ControlName } result] {
        return -code error "APSSRGapBriefScanDataFit: $result"
    }
    if [catch {exec sddsprocess $procfile -pipe=out \
                   "-redefine=col,Rate,Rate -1.0 *" \
                   -print=col,ValueString,%.12f,Rate \
                   | sddsconvert -pipe=in $ratefile -retain=col,ValueString,RatePV,FitError,BladeName \
                   -rename=col,RatePV=ControlName } result] {
        return -code error "APSSRGapBriefScanDataFit: $result"
    }
    
    if [string length $statusCallback] {
        eval $statusCallback {"Data fit done."}
    }
    return ok
}

proc APSFixScrewedXbpms {} {
    set file /home/helios/oagData/sr/IDs/badXbpmBlades.sdds
    if [catch {exec sddsproces $file -edit=col,ControlName,BladeName,ei/:RateP/ \
		   -print=col,ValueString,0 -pipe=out \
		   | sddscasr -pipe=in -restore } result] {
	return -code error "APSFixScrewedXbpms: $result"
    }
}

proc APSSRGapBriefScanLoad {args} {
    set loadRate 0
    set loadOffset 1
    set dataDir /home/helios/oagData/sr/gapScans/data/gapScanBrief
    set rootname IDscan
    set fitTolerance ""
    set statusCallback ""
    set CUSectorList ""
    set filename ""
    set pem 0
    APSStrictParseArguments {loadRate loadOffset dataDir rootname fitTolerance statusCallback CUSectorList filename pem}
    
    if ![string length $CUSectorList] {
        set  CUdataDir /home/helios/oagData/feedForwardFiles/IDgap
        if [catch {exec sddsprocess $CUdataDir/cantingLogic.sdds -pipe=out \
                     -filter=col,TreatedAsCU,1,1 \
                     | sdds2stream -pipe -col=Sector} CUSectorList] {
            return -code error "APSSRGapBriefScanLoad: $CUSectorList"
        }
    }
    if ![string length $filename] {
        global fileSelection
        set fileSelection ""
        set files [lsort -decreasing [glob -nocomplain $dataDir/${rootname}*.sdds]]
        if ![llength $files] {
            return -code error "No scan data found in $dataDir."
        }
        regsub -all $dataDir/ $files "" files0
	#the above command does not work if the directory names has / in the end.
	regsub -all $dataDir $files0 "" files0
        APSScrolledListWindow [APSUniqueName .] -name "Select a data file" \
          -label "Select a brief scan data file to install offsets and rates"  -selectMode single \
          -itemList $files0 -selectionVar fileSelection \
          -directory $dataDir -appendDescription 1 \
          -rootnameOnly 1 -descriptionName Description 
        tkwait variable fileSelection
        if ![string length $fileSelection] {
            return -code error "APSSRGapBriefScanLoad: no file was chosen for installing offsets and rates."
        }
        set filename $dataDir/[lindex [lindex $fileSelection 0] 0]
    }
    set offsetFile ${filename}.offset
    if ![file exist $offsetFile] {
        return -code error "APSSRGapBriefScanLoad: $offsetFile does not exist!"
    }
    
    set fitData [regsub {.offset} $offsetFile .fit]
    # if !$pem {
    #catch {APSSRReviewBriefScanFitData -fitData $fitData -CUSectorList $CUSectorList}
    #  }
    regsub {.offset} $offsetFile .rate rateFile
    set tmpRoot /tmp/[APSTmpString]
    if [catch {APSSRIDBadBladeSelection -filename $offsetFile} badList] {
        return -code error "APSSRGapBriefScanLoad: $badList"
    }
    
    if [llength $badList] {
        set matchOption ""
        foreach blade $badList {
            if ![string length $matchOption] {
                set matchOption -match=col,BladeName=$blade,!
            } else {
                append matchOption ,BladeName=$blade,!,&
            }
        }
        if [catch {exec sddsprocess $offsetFile  $tmpRoot.offset $matchOption -nowarnings
            exec sddsprocess $rateFile $tmpRoot.rate $matchOption -nowarnings } result] {
            return -code error "APSSRGapBriefScanLoad: $result"
        }
        set offsetFile $tmpRoot.offset
        set rateFile $tmpRoot.rate
        set fileList [glob -nocomplain $tmpRoot.*]
        APSAddToTmpFileList -ID briefscan -fileList $fileList
    }
    
    if $loadOffset { 
        set rows [exec sdds2stream $offsetFile -rows=bar]
        if !$rows {
            return -code error "APSSRGapBriefScanLoad: nothing to be installed."
        }
        if [catch {exec sddscasr -restore $offsetFile } result] { 
            return -code error "APSSRGapBriefScanLoad: $result"
        }
    }
    if $loadRate {
        if ![file exist $rateFile] {
            return -code error "APSSRGapBriefScanLoad: $rateFile does not exist"
        }
        set rows [exec sdds2stream $rateFile -rows=bar]
        if !$rows {
            return -code error "APSSRGapBriefScanLoad: nothing to be installed."
        }
        if [catch {exec sddscasr -restore $rateFile } result] {
            return -code error "APSSRGapBriefScanLoad: $result"
        }
    }
    if [llength $badList] {
        #fix bad blades through setting their rate to 0 
        if [catch {exec cavput -list=[join $badList ,] -list=:RateP=0 -pend=30} result] {
            return -code error "APSSRGapBriefScanLoad: $result"
        }
    }
}

proc APSMpSRGapBriefScan {args} {
    global execMode AllIDList APSMpSRGapBriefScan CUSectorList
    set IDList ""
    set openOtherGaps 1
    set numToAve 2
    set pause 10
    set rootname IDscan
    set lockedIDList ""
    set fixedIDList ""
    set fixedGapList ""
    set togetherList ""
    set closeInterval 1
    set loadOffset 1
    set loadRate 1
    set keepFFrunning 0
    APSStrictParseArguments {IDList openOtherGaps numToAve pause rootname lockedIDList keepFFrunning \
                         fixedIDList fixedGapList togetherList closeInterval loadOffset loadRate}
  
    set oldMode $execMode
    set ID01OffsetList ""
    if ![llength $IDList] {
        APSMpStep "Select IDs for brief scan..."
        catch {APSMakeIDGapBriefScanSelection} IDList
        if ![llength $IDList] {
            APSMpReturn error "APSMpSRGapBriefScan: No IDs selected."
        }
        set numToAve $APSMpSRGapBriefScan(numToAve)
        set pause $APSMpSRGapBriefScan(pause)
        set openOtherGaps 1
        set closeInterval $APSMpSRGapBriefScan(closeInterval)
        set waitIDList $AllIDList
        global FixedIDList FixedGapList TogetherList
        APSMpStep "Check Undulator gaps..."
        if [catch {APSSRCheckUndulatorScanType -IDList $IDList} result] {
            APSMpReturn error "APSMpSRGapBriefScan(1 - APSSRCheckUndulatorScanType): $result"
        }
        set fixedIDList $FixedIDList
        set fixedGapList $FixedGapList
        set togetherList $TogetherList
    } else {
        set waitIDList $IDList
    }
    foreach ID $fixedIDList {
        set index [lsearch $IDList $ID]
        if $index>=0 {
            set IDList [lreplace $IDList $index $index]
        }
    }
    set waitIDList [lsort -unique [concat $waitIDList $fixedIDList]]
    if [catch {exec cavget -list=[join $waitIDList ,] -list=:MotorDriveStatusM -num} valList] {
        APSMpReturn error "APSMpSRGapBriefScan(unable to read the motor status): $valList"
    }
    set len [llength $waitIDList]
    for {set i 0} {$i<$len} {incr i} {
	set val [lindex $valList $i]
	if !$val {
	    set waitIDList [lreplace $waitIDList $i $i]
	}
    }
    if [lsearch -exact $IDList ID35ds]>=0 {
        if [catch {APSCloseID35Shutter} result] {
            APSMpReturn error "APSMpSRGapBriefScan: Error in closing ID35 shutter: $result."
        }
    }
    set ID01FFID ""
    if [lsearch -exact $IDList ID01ds]>=0 {
        #need start ID01.FF when open/close the gap to make sure that the gain is zero when opening the gap;
        #and set the gain back when close the gap -- 
        APSMpStep "Start ID01.FF feedforward..."
        catch {exec sddsfeedforward /home/helios/oagData/feedForwardFiles/IDgap/ID01.FF -interval=1 -infinite&} ID01FFID
        APSMpStep "Opening ID01 ds gap to 180mm  and wait..."
        if [catch {exec cavput -list=ID01ds:GapSet=180 -pend=20
            exec cawait -waitFor=ID01ds:Gap,lowerLimit=175,upperLimit=185 -timeLimit=600 } result] {
            APSMpReturn error "APSMpSRGapBriefScan: Error opening ID01ds gap: $result"
        }
        APSMpStep "Reading ID01 blades voltage as ID01ds offset ..."
        if [catch {exec cavget -list=S1ID: -list=P1,P2 -list=: -list=A,B,C,D -list=:CorrectedM -pend=30} offsetList] {
            APSMpReturn error "APSMpSRGapBriefScan: Error opening ID01ds blades offset: $offsetList"
        }
        set ID01OffsetList [join $offsetList]
       # set ID01OffsetList {0.760409 0.760745 0.966269 0.830823 0.488806 0.576765 0.511031 0.455717}
    }
    APSMpStep "Move $IDList to 20 mm..."
    if [catch {APSSRMoveIDGaps -IDList $IDList -moveTo 20 -open $openOtherGaps \
                   -fixedIDList $fixedIDList -fixedGapList $fixedGapList -closeInterval $closeInterval \
                   -lockedIDList $lockedIDList} result] {
        APSMpReturn error "APSMpSRGapBriefScan: $result"
    }
    if [string length $ID01FFID] {
        APSMpStep "Kill ID01.FF feedforward"
        catch {exec kill -9 $ID01FFID}
    }
    #popup the ID controls adl
    # exec medm -attach -x /usr/local/iocapps/adlsys/sr/id/ID_Controls.adl &
    APSMpStep "Wait for gaps stop moving ..."
    #set waitIDList [exec sdds2stream -col=DeviceName /home/helios/oagData/sr/IDs/devices.sdds]
    
    if [catch {APSMpCheckAndWaitForIDs -deviceList $waitIDList -timeout 200} results] {
        #if {[APSPEMTKDialog .warningDialog -text "APSMpSRGapBriefScan: Problem occurred while waiting ID gaps\n$results"] == 1} {
        #    APSMpReturn error "APSMpSRGapBriefScan: $results"
        #}
	set answer [APSMultipleChoice [APSUniqueName .] -modal 1 \
			-question "Move ID gaps for brief scan failed - $result, would you like continue or abort?" \
			-returnList {1 0} \
			-labelList {Continue Abort}]
	if !$answer {
	    APSMpReturn error "APSMpSRGapBriefScan was aborted due to - $result."
	}
    }
    APSMpStep "Scan gaps ..."
    while {1} {
        while {1} {
            APSMpStep "scan $IDList..."
            if [catch {APSSRGapBriefScan -pause $pause -rootname $rootname \
                         -keepFFrunning $keepFFrunning \
                         -ID01OffsetList $ID01OffsetList \
                         -fixedIDList $fixedIDList \
                         -fiexGapList $fixedGapList \
                         -togetherList $togetherList \
                         -IDList $IDList \
                         -numToAve $numToAve } result] {
                set answer [APSMultipleChoice [APSUniqueName .] -modal 1 \
                              -question "Gap Scan failed - $result, would you like scan again or abort" \
                              -returnList {1 0} \
                              -labelList {Scan_Again Abort}]
                if !$answer { 
                    APSMpReturn error "APSMpSRGapBriefScan was aborted due to - $result."
                }
            } else {
                set outputFile $result
                APSSetVarAndUpdate status "brief gap scan collected in file $outputFile"
                break
            }
        }
        
        APSMpStep "Fit scan data for $outputFile..." 
        if ![info exist APSMpSRGapBriefScan(fit.start)] {
            set APSMpSRGapBriefScan(fit.start) 0.1
            set APSMpSRGapBriefScan(fit.factor) 340
            set APSMpSRGapBriefScan(fit.rate) -0.2
        }
        if ![info exist APSMpSRGapBriefScan(fit.tolerance)] {
            set APSMpSRGapBriefScan(fit.tolerance) 1.0
        }
        global BriefScanDialogResponse
        if [catch {APSSRGapBriefScanDataFit -inputFile $outputFile -CUSectorList $CUSectorList \
                     -tolerance $APSMpSRGapBriefScan(fit.tolerance) \
                     -fitStart $APSMpSRGapBriefScan(fit.start) \
                     -fitFactor $APSMpSRGapBriefScan(fit.factor) \
                     -fitRate $APSMpSRGapBriefScan(fit.rate)} result] {
            APSMpReturn error "APSMpSRGapBriefScan: $result"
        }
        set answer [APSMultipleChoice .info -modal 1 -name "Brief Scan" \
                        -question "What would you like to do now?" -labelList {Repeat-Scan Continue Abort} \
                        -returnList {Repeat Continue Abort} ]
        if {$answer=="Abort"} {
            APSMpReturn error "Collecting Gap FF table was aborted."
        } elseif {$answer=="Continue"} {
            break
        }
    }
    if {$loadOffset && $loadRate} {
	set msg "offsets and rates"
    } elseif {$loadOffset} {
	set msg "offsets"
    } elseif {$loadRate} {
	set msg "rates"
    } else {
	APSMpReturn ok
    }
    if [APSYesNoPopUp "Install the blades $msg from $outputFile data?"] {
        APSMpStep "loading xbpm blades $msg ..."
        if [catch {APSSRGapBriefScanLoad -loadOffset $loadOffset -loadRate $loadRate -filename $outputFile -pem 1} result] {
            APSMpReturn error "APSMpSRGapBriefScan: $result"
        }
    }
    APSMpReturn ok
}

proc APSSRReviewBriefScanFitData {args} {
    set fitData ""
    set sameScale 0
    set CUSectorList ""
    APSStrictParseArguments {fitData sameScale CUSectorList}
    
    if ![string length $fitData] {
        return -code error "APSSRReviewBriefScanFitData: fit data file not provided."
    }
    if [catch {exec sddsquery -col $fitData | grep AverageGap} gaps] {
        return -code error $gaps
    }
    if ![string length $gaps] {
        return -code error "no IDs varied in file $fitData"
    }
    
    foreach gap [lsort $gaps] {
        set sector [scan $gap ID%ld]
        lappend sectorList $sector
        regsub {AverageGap} $gap "" ID
        if [regexp {DS} $ID] {
            lappend IDList ID[format %02d $sector]ds
        } elseif [regexp {US} $ID] {
            lappend IDList ID[format %02d $sector]us 
        } else {
            lappend IDList ID[format %02d]
        }
    }
    
    if $sameScale {
        lappend optionList -same=global
    }
    set tmpRoot /tmp/[APSTmpString]
    APSAddToTmpFileList -ID gapscan -fileList $tmpRoot
    if [catch {exec sddsprocess /home/helios/oagData/sr/BPMStatus/config.sdds -pipe=out \
                 "-match=col,DeviceType=Xray ABEF" -edit=col,BPM,DeviceName,Z: \
                 -scan=col,Sector,DeviceName,S%ld,type=long -nowarnings \
                 | sddsconvert -pipe=in $tmpRoot -retain=col,Sector,BPM } result] {
        return -code error $result
    }
    set bpmList [exec sdds2stream -col=BPM $tmpRoot]
    set EFSectorList [exec sdds2stream -col=Sector $tmpRoot]
    set p1List ""
    set p2List ""
    foreach bpm $bpmList sector $EFSectorList {
        switch $bpm {
            P1 {
                lappend p1List $sector
            }
            P2 {
                lappend p2List $sector
            }
            
        }
    }
    
    foreach id $IDList sector $sectorList {
        set gap [string toupper ${id}]AverageGap
        if [lsearch $p1List $sector]>=0 {
            set p1BladeList {A1 B1 E1 F1}
        } else {
            set p1BladeList {A1 B1 C1 D1}
        }
        if [lsearch $p2List $sector]>=0 {
            set p2BladeList {A2 B2 E2 F2}
        } else {
            set p2BladeList {A2 B2 C2 D2}
        }
        if [lsearch $CUSectorList $sector]>=0 {
            if [regexp {ds} $id] {
                set bladeList $p2BladeList
            } else {
                set bladeList $p1BladeList
            }
        } else {
            set bladeList [concat $p1BladeList $p2BladeList]
        }
        set sector1 [format %02d $sector]
        foreach blade $bladeList {
            lappend plotOption -col=$gap,ID${sector1}${blade}:Corrected -graphic=sym,sub=1,type=2,scale=3 -leg=spec=data -col=$gap,ID${sector1}${blade}:CorrectedFit -grap=line -ylabel=ID${sector1}${blade} -legend=spec=fit -endp
        }
    }
    eval exec sddsplot $fitData -separate=2 -layout=2,2  $plotOption &
}

proc APSSRReviewBriefScanData {args} {
    set sameScale 0
    set scanData ""
    APSStrictParseArguments {sameScale scanData}
    
    if ![string length $scanData] {
        return -code error "APSSRReviewBriefScanData: scanData not provided"
    }
    if ![file exist $scanData] {
        return -code error "APSSRReviewBriefScanData: $scanData does not exist"
    }
    if [catch {exec sddsquery -col $scanData | grep GapSP} gaps] {
        return -code error $gaps
    } 
    if ![string length $gaps] {
        return -code error "no IDs varied in $scanData" 
    }
    if $sameScale {
        lappend optionList -same=global
    } 
    foreach gap [lsort $gaps] {  
        set sector [format %02d [scan $gap ID%ld]]
        set gap [regsub {GapSP} $gap ""]
        set gap [string toupper $gap]AverageGap
        lappend optionList -col=$gap,ID${sector}*:Corrected -ylabel=edit=%/:Corrected//
    }
    eval exec sddsplot $scanData -graph=sym,connect,subtype=1,scale=2 -separate -layout=2,2 -filename $optionList &
}

set updateBladeSelectionLevel 0
proc APSSRUpdateBladeSelection {args} {
    global updateBladeSelectionLevel
    if {$updateBladeSelectionLevel != 0} {
        set updateBladeSelectionLevel 0
        return
    } else {
        incr updateBladeSelectionLevel
    }
    set sector ""
    set item ""
    set nameFlag ""
    APSStrictParseArguments {sector item nameFlag}
    
    
    set st [string range $item end end]
    set blade S${sector}$item
    global badBlades$blade
    set blade2 ""
    set blade3 ""
    switch $st {
        A {
            regsub {:A} $blade :B blade1
	    regsub {:A} $blade :C blade2
	    regsub {:A} $blade :D blade3
        }
        B {
            regsub {:B} $blade :A blade1
	    regsub {:B} $blade :C blade2
	    regsub {:B} $blade :D blade3
        }
        C {
            regsub {:C} $blade :D blade1
	    regsub {:C} $blade :A blade2
	    regsub {:C} $blade :B blade3
	    
        }
        D {
            regsub {:D} $blade :C blade1
	    regsub {:D} $blade :A blade2
	    regsub {:D} $blade :B blade3
        }
        E {
            regsub {:E} $blade :F blade1
        }
        F {
            regsub {:F} $blade :E blade1
        }
    }
    global badBlades$blade1 badBladesCBWidget  badBlades$blade3 badBlades$blade2
    $badBladesCBWidget($blade1) invoke
    if [string length $blade2] {
	if {[set badBlades$blade2]!=[set badBlades$blade]} {
	    $badBladesCBWidget($blade2) invoke
	}
	if {[set badBlades$blade3]!=[set badBlades$blade]} {
	    $badBladesCBWidget($blade3) invoke
	}
    }
}


proc APSSRIDBadBladeSelection {args} {
    set filename ""
    APSStrictParseArguments {filename}
    
    set badList ""
    if ![string length $filename] {
        return $badList
    }
    set bladeList [exec sdds2stream -col=BladeName $filename]
    set badBlades [exec sdds2stream -col=BladeName /home/helios/oagData/sr/IDs/badXbpmBlades.sdds]
    
    set missingList ""
    set missingListVar ""
    set sectorLists ""
    for {set sector 1} {$sector<=35} {incr sector} {
        set exist 0
        foreach bpm {P1 P2} {
            foreach blade {A B C D E F} {
                set item S${sector}ID:${bpm}:$blade
                global badBlades$item
                if [lsearch $bladeList $item]<0 {
                    lappend missingVarList $item
                    lappend missingList $item
                } else {
                    if [lsearch $badBlades $item]<0 {
                        set badBlades$item 0
                    } else {
                        set badBlades$item 1
                    }
                    incr exist
                }
            }
        }
         
        lappend sectorList $sector 
    }
    
    APSDialogBox .badselection -name "Select bad blades" -cancelButton 0

    APSSRSectorButtons .check -parent .badselection.userFrame \
        -rootname badBlades -label "Select bad blades that were not fitted well in the review plots" \
        -orientation horizontal -itemList {ID:P1:A ID:P1:B ID:P1:C ID:P1:D ID:P1:E ID:P1:F ID:P2:A \
                                               ID:P2:B ID:P2:C ID:P2:D \
                                               ID:P2:E ID:P2:F} \
        -itemLabelList {P1A P1B P1C P1D P1E P1F P2A P2B P2C P2D P2E P2F} \
        -sectorControl 1 -missingList $missingList -sectorCount 35 \
      -command "APSSRUpdateBladeSelection"

    tkwait window .badselection 
    
    foreach blade $bladeList {
        if [set badBlades$blade] {
            lappend badList $blade
        }
    }
    return $badList
}
