#!/bin/sh
# \
  exec oagwish "$0" "$@"
set auto_path [linsert $auto_path 0 /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]


set CVSRevisionAuthor "\$Revision: 1.15 $ \$Author: shang $"

set status "Ready"

set monfile /home/helios/oagData/sr/BPMStats/inputFiles/bpmStats.mon
set archiveDir /home/helios/oagData/sr/BPMStats/dataFiles
set referenceFile ReferenceDatafile.sdds
set rootName bpmStats

# Quantities ending with XAVE or YAVE are in untis of counts
# Others are mm.
# The statistics are: 
#    mean value
#    standard deviation
set signalList {\
                   ms:xMean    ms:xStDev \
                   x:RawMean   x:RawStDev \
                   x.VALMean   x.VALStDev \
                   \
                   ms:yMean    ms:yStDev \
                   y:RawMean   y:RawStDev \
                   y.VALMean   y.VALStDev }

set nameList { \
                 Mean:MS:x           StDev:MS:x \
                 Mean:SRFB:x:counts  StDev:SRFB:x:counts \
                 Mean:SRFB:x         StDev:SRFB:x \
                 \
                 Mean:MS:y           StDev:MS:y \
                 Mean:SRFB:y:counts  StDev:SRFB:y:counts \
                 Mean:SRFB:y         StDev:SRFB:y }

set unitsList { \
                  mm                  micron \
                  counts              counts \
                  mm                  micron \
                  \
                  mm                  micron \
                  counts              counts \
                  mm                  micron }

set factorList { \
                  1                   1000 \
                  1                   1 \
                  1                   1000 \
                   \
                  1                   1000 \
                  1                   1 \
                  1                   1000 }


set descriptionList { \
  "Memory Scanner Horizontal Position Mean" "Memory Scanner Horizontal Position Standard Deviation"\
  "Feedback MSI Horizontal Count Mean" "Feedback MSI Horizontal Count Standard Deviation"\
  "Feedback MSI Horizontal Position Mean" "Feedback MSI Horizontal Position Standard Deviation"\
  \
  "Memory Scanner Vertical Position Mean" "Memory Scanner Vertical Position Standard Deviation" \
  "Feedback MSI Vertical Count Mean" "Feedback MSI Vertical Count Standard Deviation" \
  "Feedback MSI Vertical Position Mean" "Feedback MSI Vertical Position Standard Deviation"}


# defaults
set samples 100
set interval 0.5
set sortType StDev
set sortOrder MS
set sigmaLimit 1
set deltaMode off

set rowsToPrint 20
set printWidth 110
set printerName ""
set plotOption goodOnly

#### procedures for archiving ####

proc ReturnArchiveRootname {args} {

    APSStrictParseArguments {archiveDir}
    set dateString [exec date +%y%m%d]
    set timeString [exec date +%H%M%S]
    set pathname ${archiveDir}/${dateString}
    APSCreateDirectory -pathname $pathname

    return $pathname/$timeString
}

proc APSCreateDirectory {args} {

    APSStrictParseArguments {pathname}
    if [file isdirectory $pathname] {
        return $pathname
    } elseif [file exists $pathname] {
        error "Non-directory $pathname already exists"
    } else {
        APSCreateDirectory -pathname [file dirname $pathname]
        exec mkdir $pathname
        exec chmod a+w $pathname
        return $pathname
    }
}

proc MakeListOfArchives {args} {
    global status

    APSStrictParseArguments {rootDir pattern}
    set allFiles [glob -nocomplain $rootDir/*]
    set dirList ""
    foreach dir $allFiles {
        if [file isdirectory $dir] {
            lappend dirList [APSStringTrimLeft $dir $rootDir]
        }
    }
    set fileList ""
    foreach dir $dirList {
        set files [glob -nocomplain $rootDir/$dir/$pattern]
        if {[llength $files] != 0} {
            append fileList "$files "
        }
    }
    set fileList [join $fileList]
    set newList ""
    for {set i 0} {$i < [llength $fileList]} {incr i} {
        set item [lindex $fileList $i]
        set item [APSStringTrimLeft $item $rootDir]
        set item [string range $item 0 [expr [string first - $item] - 1]]
        set newItem [string trimleft $item]
        if {[lsearch $newList $newItem] == -1} {
            lappend newList $newItem
        }
    }
    set newList [DateSort $newList]
    return $newList
}

proc plotArchivedData {args} {
    global status archiveDir rootName
    global referenceFile
    APSStrictParseArguments {deltaMode}

    set archiveList [MakeListOfArchives -rootDir $archiveDir -pattern "??????-${rootName}.sdds"]
    set archive [APSChooseItemFromList -itemList $archiveList \
                   -name "Select Archive to Plot" -returnIndices 0 -multiItem 0]
    if {$archive == ""} {
        set status "No file selected."
        return
    }
    switch $deltaMode {
        reference {
            set baseline $archiveDir/$referenceFile
        }
        select {
            set deltaArchive [APSChooseItemFromList -itemList $archiveList \
                                -name "Select Baseline Archive" -returnIndices 0 -multiItem 0]
            if {$deltaArchive == ""} {
                set status "No reference file selected."
                return
            }
            set baseline $archiveDir/${deltaArchive}-${rootName}.sdds
        }
        default {
            set baseline ""
        }
    }
    
    set status "Plotting archive $archiveDir/$archive..."; update
    plotData -filename $archiveDir/${archive}-${rootName}.sdds -baseline $baseline

}

#### All other procedures ####

proc plotData {args} {
    global status nameList descriptionList unitsList archiveDir rootName plotOption
    global printerName

    APSStrictParseArguments {filename baseline}
    if {$plotOption=="all"} {
        set tmpfilename $filename
    } else {
        set tmpfilename [CleanUpBpms $filename]
    }
    if {$baseline != ""} {
        if {[file exists $baseline] && [file exists $filename]} {
            if {$plotOption=="goodOnly"} {
                set tmpBaseline /tmp/[APSTmpString].base
                APSAddToTmpFileList -ID SRBpmStats -fileList $tmpBaseline
                if [catch {exec sddsselect $baseline $tmpfilename -match=BPMName -reuse $tmpBaseline } result] {
                    return -code error $result
                }
            } else {
                set tmpBaseline $baseline
            }
            set tmpPlot /tmp/[APSTmpString]
            if [catch {exec sddschanges $tmpfilename -pipe=out -baseline=$tmpBaseline \
                         -copy=Sector -changesIn=[join $nameList ,] \
                         | sddsconvert -pip=in $tmpPlot -edit=col,ChangeIn*,%/ChangeIn// } result] {
                set status "Error in sddschanges: $result"
                return
            }
            set refArchive [APSStringTrimLeft $baseline $archiveDir]
            set refArchive [APSStringTrimRight $refArchive .sdds]
        }
    }
    
    if [file exists $filename] {
        set archive [APSStringTrimLeft $filename $archiveDir]
        set archive [APSStringTrimRight $archive -${rootName}.sdds]
        if {$baseline != ""} {
            set cmd "sddsplot -sep $tmpPlot -lay=1,2"
        } else {
            set cmd "sddsplot -sep $tmpfilename -lay=1,2"
        }
        
        lappend cmd "-xlabel=Sector"
        lappend cmd \"-title=\"
        
        set plots 0
        foreach description $descriptionList name $nameList units $unitsList {
            lappend cmd \"-col=Sector,$name\"
            lappend cmd \"-ylabel=Value ($units)\"
            lappend cmd -grap=imp,typ=2
            if {$baseline != ""} {
                lappend cmd \"-topl=$description (w.r.t baseline)\"
            } else {
                lappend cmd \"-topl=$description\"
            }		    
            if {[expr fmod($plots,2)] == 1.0} {
                lappend cmd \"-string=Archive: $archive,p=0.78,q=-0.16,scale=0.8\"
                if {$baseline != ""} {
                    lappend cmd \"-string=Baseline: $refArchive,p=-0.15,q=-0.16,scale=0.8\"
                }
                lappend cmd -endp
                incr plots
            }
       }
        set status "Launching sddsplot..."
        puts $cmd
        if [catch {eval exec [join $cmd] & } result] {
            set status "Error in sddsplot: $result"
            return
        }
        set status "Done"
    } else {
        set status "Nothing to plot"
    }
}
proc CleanUpBpms {filename} {
    #remove bad bpms that are not ok for RTFB 
    set statusfile /home/helios/oagData/sr/BPMStatus/config.sdds
    set newFile /tmp/[APSTmpString]
    APSAddToTmpFileList -ID SRBpmStats -fileList "$newFile.good $newFile.archive"
    if [catch {exec sddsprocess $statusfile -pipe=out -filter=col,OkForRTFeedbackH,1,1,OkForRTFeedbackV,1,1,& \
                 | sddsconvert -pipe=in -retain=col,DeviceName $newFile.good
        exec sddsselect $filename $newFile.good $newFile.archive -reuse \
                 -match=BPMName=DeviceName } result] {
        return -code error $result
    }
    return $newFile.archive
}


proc collectData {} {
    global status samples interval archiveDir rootName OAGGlobal
    global signalList nameList unitsList factorList 
    global monfile 

    if ![APSYesNoPopUp "Confirm: collect data now?"] {
        return
    }
    set tmpfile /tmp/[APSTmpString]
    set filename [ReturnArchiveRootname -archiveDir $archiveDir ]-${rootName}.sdds
    set status "Collecting data...[exec date +%H:%M:%S]"; update
    if [catch {exec sddsstatmon $monfile -erase -oncaerror=skip -ezca=0.01,1000 \
                 -steps=1 -interval=${interval} -sample=$samples \
                 -include=mean,standard ${tmpfile}.raw \
                 -getUnits=ifNoneGiven} result] {
        set status "collectData: Error with sddsstatmon: $result"
        return
    }

    set status "Processing $filename ...[exec date +%H:%M:%S]"; update
    # remove the SRFB: prefix if it appears in a PV.
    # this is a complicated way to group the columns. We can't use
    # sddscollect since there are an unequal number of columns
    # that match the signal types.
    foreach signal $signalList units $unitsList factor $factorList {
        if [catch {exec sddsconvert ${tmpfile}.raw  -pipe=out \
                     "-del=col,*" \
                     "-retain=col,*$signal"\
                     | sddstranspose -pipe -oldC=BPMName -root=$signal \
                     | sddsprocess -pipe=in ${tmpfile}.$signal \
                     "-redef=col,$signal,$signal $factor *,units=$units" \
                     -reedit=col,BPMName,%/SRFB://a2s/:/b30d \
                 } result] {
            set status "collectData: Error with sddsconvert: $result"
            return
        }
    }
    

    # The first file must have the complete set of bpms.
   # set firstFile ${tmpfile}.[lindex $signalList 0]
    #the first file does not contain FPGA bpms, while RawMean has; to have FPGA bpm data, can not use the first file
    set firstFile ${tmpfile}.x:RawMean
    set fileList ""
   # foreach signal [lrange $signalList 1 end] {
   #     lappend fileList ${tmpfile}.$signal
   # }
    foreach signal $signalList {
        if {$signal!="x:RawMean"} {
            lappend fileList ${tmpfile}.$signal
        }
    }
    set renameOptions ""
    foreach signal $signalList name $nameList {
        lappend renameOptions "-rename=col,$signal=${name}"
    }
    # convert the standard deviation data into microns
    if [catch {eval exec sddsxref $firstFile $fileList -pipe=out \
                 -match=BPMName -fillin -noWarning \
                 | sddsprocess -pipe \
                 -print=par,MonitorFilename,$monfile \
                 -def=par,Interval,$interval \
                 | sddsxref -pipe $OAGGlobal(SRLatticesDirectory)/default/aps.twi \
                 -match=BPMName=ElementName -take=s -noWarning \
                 | sddsprocess -pipe \
                 \"-def=col,Sector,s 27.6 / \" \
                 | sddssort -pipe -col=s \
                 | sddsconvert -pipe=in $filename \
                 -delete=par,InputFile,PageTimeStamp \
                 -del=col,s \
                 $renameOptions \
             } result] {
        set status "collectData: Error in sddsxref: $result"
        return
    }
    
    if [catch {exec chmod a-w $filename} result] {
        set status "collectData: Error with chmod: $result"
    }
    
    set fileList [glob ${tmpfile}.*]
    if {[llength $fileList] != 0} {
     #   eval file delete $fileList
    }
    plotData -filename $filename -baseline ""
}

proc printoutData {args} {
    global status archiveDir rootName 
    global rowsToPrint printWidth printerName
    set tmpfile /tmp/[APSTmpString]

    APSStrictParseArguments {sortOrder sortType sigmaLimit}
    
    set archiveList [MakeListOfArchives -rootDir $archiveDir -pattern "??????-${rootName}.sdds"]
    set archive [APSChooseItemFromList -itemList $archiveList \
                   -name "Select Archive to Plot" -returnIndices 0 -multiItem 0]
    if {$sigmaLimit=="goodBpms"} {
        set printfile [CleanUpBpms $archiveDir/${archive}-${rootName}.sdds]
    } else {
        set printfile $archiveDir/${archive}-${rootName}.sdds
    }
    if {![file exists $printfile]} {
        set status "Nothing to print out"
        return
    }
    set status "Preparing to print out archive $archive..."; update
    
    if {$sigmaLimit != "none" && $sigmaLimit!="goodBpms" } {
        set minNorm [expr 1 / exp($sigmaLimit)] 
    } else {
        set minNorm 0.0
    }
    
    if {$printerName == ""} {
        set printCmd "enscript -fCourier8"
    } else {
        set printCmd "enscript -fCourier8 -P$printerName"
    }

    foreach plane {x y} {
        if {$plane == "x"} {
            set planeString "Horizontal BPMs"
        } else {
            set planeString "Vertical BPMs"
        }
        # sortOrder has possible values of MS SRFB NAME
        # sortType has possible values of Mean StDev
        switch $sortOrder {
            NAME {
                set sorting ${sortType}:MS:${plane}
            }
            MS - 
            SRFB -
            default {
                set sorting ${sortType}:${sortOrder}:${plane}
            }
        }
        
        set mmformat %6.3f
        set micronformat %5.1f

        # sortOrder has possible values of MS SRFB NAME
        # sortType has possible values of Mean StDev
        switch $sortOrder {
            NAME {
                set sortoption -col=Sector
                set sortdesc BPMName
            }
            MS -
            SRFB -
            default {
                set sortoption -col=normValue,decr
                set sortdesc $sorting
            }
        }
        # Presently the possibility of printing out the counts data has
        # eben removed.
        if [catch {exec sddsprocess $printfile -pipe=out \
                     "-def=col,normValue,$sorting abs" \
                     -process=normValue,max,MaxValue \
                     -process=normValue,min,MinValue \
                     "-redef=col,normValue,normValue MinValue - MaxValue MinValue - / " \
                     | sddsprocess -pipe -filter=col,normValue,$minNorm,1.0 \
                     | sddssort -pipe $sortoption \
                     | sddsprintout -pipe=in ${tmpfile}.print$plane \
                     "-title=Printout from SRBpmStats Tool\n----------------------------------\n\nArchive: $archive ($planeString)\nSorted by: ${sortdesc}\nSigmal Limit on Printout: $sigmaLimit\n" \
                     -format=string=%8s \
                     "-col=BPMName,format=%8s" \
                     "-col=Mean:*:${plane},format=${mmformat},edit=a%/:counts//" \
                     "-col=StDev:*:${plane},format=${micronformat},edit=a%/:counts//" } result] {
            set status "Error with sddssort: $result"
        }
        set windowName [APSUniqueName win]
        APSFileDisplayWindow .$windowName -fileName ${tmpfile}.print$plane \
          -deleteOnClose 1 -comment "Bpm Statistics for Plane $plane" \
          -width $printWidth -height [expr $rowsToPrint + 4] \
          -sddsExportableFile $printfile -printCommand $printCmd
    }
    set status "Done"
}

proc DateSort {list} {
    # this if for sorting a list of files or directory names
    # that start with 99... and 00...
    # we want to put the 00 first
    set list00 ""
    set list99 ""
    foreach item $list {
        if {[string index $item 1] == "9"} {
            lappend list99 $item
        } else {
            lappend list00 $item
        }
    }
    return [concat [lsort -decreasing $list00] [lsort -decreasing $list99]]
}

####### MAIN #######

APSApplication . -name "SR BPM Statistics Tool" \
  -version $CVSRevisionAuthor \
  -overview "Collect/review  statistics on storage ring bpms" \
  -contextHelp "Collect/review statistics on storage ring bpms"

APSScrolledStatus .status -parent .userFrame -packOption "-side top" -textVariable status -width 65

APSFrame .one -parent .userFrame -packOption "-fill x"

APSButton .collectbutton -parent .userFrame.one.frame -text "Collect Data" \
  -packOption "-side left" -command { collectData } \
  -contextHelp "Collect the specified number of samples for all bpms via EPICS at specified time intervals.\
 Data is subsequently processed and plotted. The data is archived for future retrieval."

APSLabeledEntry .samplesEntry -parent .userFrame.one.frame -width 5 \
  -packOption "-side left" -label "Samples" -textVariable samples \
  -contextHelp "Number of samples to be collected for generating statistics."

APSLabeledEntry .intervalEntry -parent .userFrame.one.frame -width 5 \
  -packOption "-side left" -label "Interval(s)" -textVariable interval \
  -contextHelp "Time interval between samples."

APSFrame .two -parent .userFrame -packOption "-fill x"

APSButton .plotbutton -parent .userFrame.two.frame -text "Plot Archived Data" \
  -packOption "-side left" -command { plotArchivedData -deltaMode $deltaMode} \
  -contextHelp "Plot data from a previous archive. The archive is chosen from a \
list of available data archives."
APSFrame .plotoption -parent .userFrame.two.frame -packOption "-side right"
.userFrame.two.frame.plotoption.frame configure -bd 0 -relief flat
APSRadioButtonFrame .allgood -parent .userFrame.two.frame.plotoption.frame -label "" \
  -packOption "-side top" -orientation horizontal -variable plotOption \
  -buttonList {"all bpms" "good bpms only"} -valueList {all goodOnly} \
  -contextHelp "choose to plot all bpms or only the bpms that are OK for RTFB."

APSRadioButtonFrame .deltaChoice -parent .userFrame.two.frame.plotoption.frame \
  -packOption "-side top" -orientation horizontal -label "Delta Mode:" \
  -variable deltaMode -buttonList {"Off" "Reference" "Choose" } \
  -valueList {off reference select} \
  -contextHelp "Choose whether to plot relative to reference data or to a selected file.\
Off: plot the raw data; Reference: use the pre-defined reference file;\
  Choose: select the file to be used as a reference."

APSFrame .three -parent .userFrame -packOption "-fill x" -label ""

APSButton .printoutbutton -parent .userFrame.three.frame -text "Printout Archived Data" \
  -packOption "-side left" \
  -command {printoutData -sortOrder $sortOrder -sortType $sortType -sigmaLimit $sigmaLimit}

APSLabeledEntry .printerNameEntry -parent .userFrame.three.frame -width 6 \
  -packOption "-side right" -label "Printer:" -textVariable printerName \
  -contextHelp "Enter the name of the printer to use for printouts.\
If the field is empty, the default printer is used."

APSFrame .four -parent .userFrame -relief flat -packOption "-fill x"

APSRadioButtonFrame .sortChoice -parent .userFrame.four.frame \
  -packOption "-side left" -orientation horizontal -label "Sort by:" \
  -variable sortOrder -buttonList {"Memory Scanner" "srfb MSI" "name" } \
  -valueList {MS SRFB NAME} \
  -contextHelp "Chose the sort criteria for the printout."

APSRadioButtonFrame .sortType -parent .userFrame.four.frame \
  -packOption "-side left" -orientation horizontal -label "" \
  -variable sortType -buttonList {"Mean" "StDev" } \
  -valueList {Mean StDev} \
  -contextHelp "Chose the sort criteria for the printout."

APSRadioButtonFrame .sigmaChoice -parent .userFrame \
  -packOption "-side left" -orientation horizontal -label "Sigmas to print:" \
  -variable sigmaLimit -buttonList {"1-sigma" "2-sigma" "3-sigma" "all bpms" "good bpms"} \
  -valueList {1 2 3 none goodBpms} \
  -contextHelp "Select the number of standard deviations (n-sigma) from the mean of the bpm  data distribution for removal from the printout. The removed points are  considered outliers of the distribution. There is no outlier filtering for \"all bpms\" and \"good bpms\". \"good bpms\" selects the bpms that are OK for RTFB."

