#!/bin/sh  
# \
exec oagwish "$0" "$@"


set auto_path [linsert $auto_path 0 /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]
APSDebugPath
set CVSRevisionAuthor "\$Revision: 1.7 $ \$Author: emery $"

set originalDir [pwd]

APSStandardSetup

set args $argv
set group Main
APSStrictParseArguments {group}

set MonDataDirRoot /home/helios/oagData

set MonDataName {}
set MonDataDirList {}
set MonDataRootnameList {}
set DataGroupCommandList {}
set DataGroupValueList {}
set NumberOfDataGroups 0

proc AddToMonDataLists {name dir rootname args} {
    global MonDataName MonDataDirList MonDataRootnameList
    global DataGroupCommandList DataGroupValueList NumberOfDataGroups 
    lappend MonDataName "$name"
    lappend MonDataDirList $dir
    lappend MonDataRootnameList $rootname
    lappend DataGroupCommandList ""
    lappend DataGroupValueList $NumberOfDataGroups
    incr NumberOfDataGroups
}
set MonDataPresetPlotDir /home/helios/oagData/monitoring/MonitorDataReviewFiles

set limitPerRow 16
AddToMonDataLists  "SR rf"  monitoring/SRrfg  SRrf  
AddToMonDataLists  "SR Absorber H2O"  monitoring/SRH2O  SRH2O  
AddToMonDataLists  "SR vacuum"  monitoring/SRvac  SRvac  
AddToMonDataLists  "SR chamber temps" monitoring/SRChamberTemp SRChamberTemp
AddToMonDataLists  "SR DCPS: correctors/dipole" logging/SRDCPS-HVD  SRDCPS-HVD
AddToMonDataLists  "SR DCPS: correctors/dipole\n(extensive)" logging/SRDCPS-HVD-Extensive  SRDCPS-HVDE
AddToMonDataLists  "SR DCPS: quads/sextupoles" logging/SRDCPS-QS  SRDCPS-QS
AddToMonDataLists  "SR DCPS: quads/sextupoles\n(extensive)" logging/SRDCPS-QS-Extensive  SRDCPS-QSE
AddToMonDataLists  "SR Pulsed Power Supplies" monitoring/SRPulsed SRPulsed 
AddToMonDataLists  "Process Water/Air Temps" monitoring/processWater processWater 
AddToMonDataLists  "Power System" monitoring/powerSystem powerSystem 
AddToMonDataLists  "PS/Mag H20 Flow/Pres." monitoring/SRPSMagH2O SRPSMagH2O
AddToMonDataLists  "SR BPMs" logging/srBPMs srBPMs 
AddToMonDataLists  "SR Ave. BPMs" logging/srBPMAve srBPMAve 
AddToMonDataLists  "SR Fast Logger" monitoring/SRFastLog1 SRFastLog1 
AddToMonDataLists  "SR Synch. Light Mon." monitoring/SRSynchLightMon SRSynchLightMon
AddToMonDataLists  "SR Hydrostatic Level" monitoring/HSLevel HSLevel
AddToMonDataLists  "SR Switchgear" monitoring/SRSG SRSG
AddToMonDataLists  "SR Injection" monitoring/SRInjection SRInjection
AddToMonDataLists  "SR Feedback Status" monitoring/RTFBStatus RTFBStatus
AddToMonDataLists  "SR Feedback Corrector Errors" monitoring/RTFBCorrectorErrors RTFBCorrectorErrors
AddToMonDataLists  "SR Thermocouples" monitoring/SRThermocouples SRThermocouples
AddToMonDataLists  "ID data" monitoring/IDs IDs  
AddToMonDataLists  "BM data" monitoring/BMs BMs 
AddToMonDataLists  "Beamline shutters" monitoring/shutters shutters 
AddToMonDataLists  "Beam Charge (LTP to SR)"  monitoring/Charge  ChargeFast  
AddToMonDataLists  "Beam Charge (Gun to PTB)"  monitoring/Current  Current  
AddToMonDataLists  "BTS Power Supplies" monitoring/BTSPS BTSPS
AddToMonDataLists  "Booster Pulsed Power Supplies" monitoring/BoosterPulsed BoosterPulsed
AddToMonDataLists  "Booster Vacuum"  monitoring/boosterVac  boosterVac  
AddToMonDataLists  "Booster rf" monitoring/BRF BRF 
AddToMonDataLists  "Booster Ramp Parameters" monitoring/boosterRampParam boosterRampParam 
AddToMonDataLists  "Booster Injection" monitoring/BInjection BInjection
AddToMonDataLists  "PAR/LET Vacuum"  monitoring/parVac  parVac  
AddToMonDataLists  "PAR/LET DC Power Supplies" monitoring/PLETDCPS PLETDCPS 
AddToMonDataLists  "PAR Pulsed Power Supplies" monitoring/PulsedMag  PulsedMag  
AddToMonDataLists  "PAR RF1"  monitoring/PRF1  PRF1  
AddToMonDataLists  "PAR RF12"  monitoring/PRF12  PRF12  
AddToMonDataLists  "Linac Vacuum" monitoring/linacVacuum linacVacuum 
AddToMonDataLists  "Linac Power Supplies" monitoring/linacPS linacPS 
AddToMonDataLists  "Linac RF" monitoring/linacRF linacRF 
AddToMonDataLists  "Linac Modulators" monitoring/linacModulators linacModulators 
AddToMonDataLists  "Linac Gun" monitoring/gun gun 
AddToMonDataLists  "Linac Diag" logging/linacDiag linacDiag
AddToMonDataLists  "Linac Switch Gear" monitoring/linacSG linacSG
AddToMonDataLists  "RF Gun" monitoring/rfgun rfgun
AddToMonDataLists  "injector IOCs" monitoring/injIOC injIOC 
AddToMonDataLists  "SR IOCs" monitoring/srIOC srIOC 
AddToMonDataLists  "Mobile MV200" monitoring/MobileMV200 MMV200
AddToMonDataLists  "Mobile MV200 Vid3" monitoring/MobileMV200Vid3 MMV200Vid3
AddToMonDataLists  "Weather" ANLWeather  met 

set MonDataGroupIndex 0

proc MakeDateTimeFrame {widget args} {
    set parent .
    APSStrictParseArguments {parent}
    set label "Date/Time of Interest"
    APSFrame $widget -parent $parent -label $label
    set w $parent$widget.frame

    global BaselineDuration ROIDuration
    set BaselineDuration 60
    set ROIDuration 60

    APSDateTimeAdjEntry .baselineDate -parent $w \
      -yearVariable BaselineYear \
      -monthVariable BaselineMonth \
      -dayVariable BaselineDay \
      -hourVariable BaselineHour \
      -label "Baseline date/time (year, month, day, hour): " -defaultHour 0 \
      -command ResetMonDataFileList
    APSLabeledEntry .baselineMin -parent $w \
      -textVariable BaselineDuration -label "Baseline duration (minutes): " -contextHelp \
      "Enter the duration for baseline data."
    APSDateTimeAdjEntry .aROIDate -parent $w \
      -yearVariable ROIYear \
      -monthVariable ROIMonth \
      -dayVariable ROIDay \
      -hourVariable ROIHour \
      -label "Region-of-interest date/time (year, month, day, hour):   " -defaultHour 24 \
      -command ResetMonDataFileList
    APSLabeledEntry .aROIMin -parent $w \
      -textVariable ROIDuration -label "Region-of-interest duration (minutes): " -contextHelp \
      "Enter the duration for region-of-interest."

    SetDateTimeToToday -rootname Baseline -hour 0
    SetDateTimeToToday -rootname ROI \
      -hour [clock format [expr [clock seconds]-60*60] -format %H:%M]
}

proc SetDateTimeToToday {args} {
    set rootname ""
    set hour 0
    ResetMonDataFileList
    APSStrictParseArguments {rootname hour}

    global ${rootname}Month ${rootname}Year ${rootname}Day ${rootname}Hour
    
    APSDateBreakDown -dayVariable ${rootname}Day -yearVariable ${rootname}Year \
      -monthVariable ${rootname}Month -twoDigitYear 0 -leadingZeros 0
    set ${rootname}Hour $hour
}

    
set MonDataStatus Ready.
proc SetMonDataStatus {text} {
    global MonDataStatus
    set MonDataStatus $text
    update
}

set DataDir ""

proc ResetMonDataFileList {} {
    global MonDataFileList MonDataFileListIsOld
    global MonDataGroupIndex

    set MonDataFileList {}
    set MonDataFileListIsOld 1
}

proc MakeGroupSelectionFrame {widget args} {
    set parent ""
    APSParseArguments {parent}
    
    global MonDataName DataGroupValueList MonDataGroupIndex
    global DataGroupCommandList  limitPerRow

    APSRadioButtonFrame $widget -parent $parent -label "Data group" \
      -variable MonDataGroupIndex \
      -buttonList "$MonDataName" \
      -valueList $DataGroupValueList \
      -orientation vertical \
      -contextHelp "Choose the data group that you want to view data from." \
      -commandList $DataGroupCommandList \
      -limitPerRow $limitPerRow
}

proc CompareData {args} {
    global group 
    global MonDataGroupIndex
    global MonDataRootnameList MonDataDirList MonDataDirRoot
    global filterSignal filterLow filterHigh

    set mode ""
    APSStrictParseArguments {mode}

    set rootname [lindex $MonDataRootnameList $MonDataGroupIndex]
    set dataDir  $MonDataDirRoot/[lindex $MonDataDirList $MonDataGroupIndex]

    foreach type {Baseline ROI} {
        foreach period {Year Month Day Hour Duration} {
            global ${type}${period}
        }
        set ${type}TimeOfDay [APSConvertTimeToHours [set ${type}Hour]]
    }

    set BaselineFileList \
      [APSFindFilesBetweenDates \
         -rootname ${rootname}- \
         -directory $dataDir \
         -startDateList [APSFormatDate -year $BaselineYear -month $BaselineMonth \
                           -day $BaselineDay -dateFormat list] \
         -endDateList [APSFormatDate -year $BaselineYear -month $BaselineMonth \
                         -day $BaselineDay -dateFormat list] ]
    if ![llength $BaselineFileList] {
        return -code error "No baseline files found"
    }
    set availableSignalList [APSGetSDDSNames -class column -fileName [lindex $BaselineFileList 0]]

    set ROIFileList \
      [APSFindFilesBetweenDates \
         -rootname ${rootname}- \
         -directory $dataDir \
         -startDateList [APSFormatDate -year $ROIYear -month $ROIMonth \
                           -day $ROIDay -dateFormat list] \
         -endDateList [APSFormatDate -year $ROIYear -month $ROIMonth \
                         -day $ROIDay -dateFormat list] ]
    if ![llength $ROIFileList] {
        return -code error "No ROI files found"
    }

    set filterSignal [string trim $filterSignal]
    if {[string length $filterSignal] && \
          $filterLow<$filterHigh && \
          [lsearch -exact $availableSignalList $filterSignal]==-1} {
        return -code error "Signal $filterSignal not in data file---can't use to filter."
    }

    set w [APSUniqueName .]
    set signalList [APSChooseItemFromList -name "CompareDataLogs data selection" \
                      -itemList $availableSignalList -returnList $availableSignalList \
                      -multiItem 1]
    if [llength $signalList]==0 return


    set tmpRoot /tmp/[APSTmpString]
    APSAddToTmpFileList -ID CDL -fileList "$tmpRoot.base $tmpRoot.basef $tmpRoot.ROI $tmpRoot.ROIf"
    if [catch {createStatsFile -fileList "$BaselineFileList" -output $tmpRoot.base \
                 -hour $BaselineTimeOfDay -fullOutput $tmpRoot.basef \
                 -duration $BaselineDuration -suffix Baseline -signalList $signalList \
                 -filterList [list $filterSignal $filterLow $filterHigh]
        createStatsFile -fileList "$ROIFileList" -output $tmpRoot.ROI -hour $ROITimeOfDay \
                 -duration $ROIDuration -suffix ROI -fullOutput $tmpRoot.ROIf \
                 -signalList $signalList -filterList \
                [list $filterSignal $filterLow $filterHigh] } result] {
        return -code error "$result"
    }

    set baselineRows [exec sdds2stream -rows=bare $tmpRoot.basef]
    set roiRows [exec sdds2stream -rows=bare $tmpRoot.ROIf]
    APSSetVarAndUpdate MonDataStatus "Baseline region: $baselineRows samples"
    APSSetVarAndUpdate MonDataStatus "ROI region: $roiRows samples"

    switch $mode {
        compareMeans {
            APSAddToTmpFileList -ID CDL -fileList "$tmpRoot.comp $tmpRoot.printout"
            if [catch {compareMeans -baseline $tmpRoot.base -ROI $tmpRoot.ROI -outputRoot $tmpRoot} result] {
                APSDeleteTmpFileList -ID CDL
                return -code error "$result"
            }
        }
        countUnlikelyValues {
            if [catch {countUnlikelyValues -baseline $tmpRoot.basef -ROI $tmpRoot.ROIf \
                         -outputRoot $tmpRoot} result] {
                APSDeleteTmpFileList -ID CDL
                return -code error "$result"
            }
        }
        compareHistograms {
            set points1 $baselineRows
            set points2 $roiRows
            set bins [expr int($points1/5.0)]
            if [expr int($points2/5.0)]<$bins {
                set bins [expr int($points2/5.0)]
            }
            if $bins<10 {
                set bins 10
            }
            APSAddToTmpFileList -ID CDL -fileList "$tmpRoot.basehis $tmpRoot.ROIhis"
            if [catch {exec sddsmultihist -bins=$bins $tmpRoot.basef $tmpRoot.basehis \
                         -columns=[join $signalList ,] -separate 
                exec sddsmultihist -bins=$bins $tmpRoot.ROIf $tmpRoot.ROIhis \
                         -columns=[join $signalList ,] -separate} result] {
                APSDeleteTmpFileList -ID CDL
                return -code error "$result"
            }
            set columnList [exec sddsquery -column $tmpRoot.ROIhis]
            foreach column $columnList {
                if ![string match *Frequency $column] {
                    lappend plotOpt -column=$column,${column}Frequency 
                }
            }
            eval exec sddsplot -groupby=names -sep=names \
              $plotOpt $tmpRoot.basehis -legend=spec=baseline \
              $plotOpt $tmpRoot.ROIhis -legend=spec=ROI -graph=line,type=1 &
        }
        default {
        }
    }
    after 2000 "APSDeleteTmpFileList -ID CDL"
}

proc countUnlikelyValues {args} {
    set baseline ""
    set ROI ""
    set outputRoot ""
    APSStrictParseArguments {baseline ROI outputRoot}

    if {![string length $baseline] || ![file exists $baseline]} {
        return -code error "countUnlikelyValues: baseline file not given or nonexistent"
    }
    if {![string length $ROI] || ![file exists $ROI]} {
        return -code error "countUnlikelyValues: ROI file not given or nonexistent"
    }
    
    if [catch {exec sddsxref $ROI $baseline -pipe=out -leave=* -transfer=param,* \
                 | sddsprocess -pipe \
                 "-define=column,%sRCheck,%s %sBaselineP10 - %sBaselineDRange 1e-16 + /,select=*" \
                 | sddsprocess -pipe \
                 -process=*RCheck,count,%sCount,bottom=0,top=1 \
                 -define=parameter,Rows,n_rows,type=long \
                 | sddscollapse -pipe \
                 | sddsprocess -pipe \
                 "-define=column,%sFrac,1 %s Rows / -,select=*RCheckCount" \
                 | sddscollect -pipe \
                 -collect=suffix=RCheckCount,column=InsideCount -collect=suffix=RCheckCountFrac,column=OutsideFrac \
                 | sddssort -pipe -column=OutsideFrac,decr \
                 | sddsprintout -column=Rootname,label=ReadbackName,format=%30s \
                    "-title=Values outside 10%-90% region of baseline distribution\n" \
                    -column=OutsideFrac -pipe=in $outputRoot.printout} result] {
        return -code error "countUnlikelyValues: $result"
    }
    APSFileDisplayWindow [APSUniqueName .] -fileName $outputRoot.printout \
      -width 120
}

proc compareMeans {args} {
    set baseline ""
    set ROI ""
    set outputRoot ""
    APSStrictParseArguments {baseline ROI outputRoot}

    if {![string length $baseline] || ![file exists $baseline]} {
        return -code error "compareMeans: baseline file not given or nonexistent"
    }
    if {![string length $ROI] || ![file exists $ROI]} {
        return -code error "compareMeans: ROI file not given or nonexistent"
    }
    if [catch {exec sddsxref $baseline $ROI -pipe=out \
                 -match=Rootname -take=* -nowarning \
                 | sddsprocess -pipe \
                 "-define=column,Deviation,BaselineAve ROIAve -" \
                 "-define=column,DeviationSigma,BaselineSigma sqr ROISigma sqr + sqrt" \
                 "-define=column,SigmaNormAbsDeviation,Deviation DeviationSigma 1e-16 + / abs" \
                 | sddssort -pipe -column=SigmaNormAbsDeviation,decr \
                 | tee $outputRoot.comp \
                 | sddsprintout -pipe=in $outputRoot.printout \
                 "-title=Comparision of mean values from two times\n" \
                 -column=Rootname,format=%32s -column=BaselineAve,format=%10.3g,label=Baseline \
                 -column=BaselineSigma,format=%10.3g,label=Sigma \
                 -column=Deviation,format=%10.3g \
                 -column=DeviationSigma,format=%10.3g \
                 -column=SigmaNormAbsDeviation,format=%10.3g } result] {
        return -code error "compareMeans: $result"
    }
    APSFileDisplayWindow [APSUniqueName .] -fileName $outputRoot.printout \
      -sddsExportableFile $outputRoot.comp -width 120
}

proc createStatsFile {args} {
    set fileList ""
    set output ""
    set fullOutput /dev/null
    set hour 0
    set duration 0
    set suffix Baseline
    set signalList *
    set filterList ""
    APSStrictParseArguments {fileList output hour duration suffix fullOutput signalList filterList}
    if ![llength $fileList] {
        return -code error "createStatsFile: no files listed"
    }
    if ![string length $output] {
        return -code error "createStatsFile: no output file given"
    }
    if {$hour<0 || $hour>24} {
        return -code error "createStatsFile: invalid hour: $hour"
    }
    if {$duration<=0} {
        return -code error "createStatsFile: invalid duration: $duration"
    }
    set filterSignal ""
    set filterLow 0
    set filterHigh 0
    if [llength $filterList]==3 {
        APSSetVarsFromList -variableList {filterSignal filterLow filterHigh} \
          -valueList $filterList
    }
    set filterOption ""
    if {$filterLow<$filterHigh && [string length $filterSignal]} {
        set filterOption -filter=column,$filterSignal,$filterLow,$filterHigh
        lappend signalList $filterSignal
    }
    if [catch {eval exec sddscombine -merge $fileList -pipe=out -recover \
                 -retain=column,[join $signalList ,],TimeOfDay \
                 | sddsprocess -pipe \
                 -filter=column,TimeOfDay,$hour,[expr $hour+$duration/60.0] \
                 $filterOption \
                 | sddsprocess -pipe -delete=col,TimeOfDay \
                 -process=*,ave,%s${suffix}Ave \
                 -process=*,perc,%s${suffix}P10,perc=10 \
                 -process=*,drange,%s${suffix}DRange \
                 -process=*,sigma,%s${suffix}Sigma \
                 -process=*,stand,%s${suffix}StDev \
                 | tee $fullOutput \
                 | sddscollapse -pipe \
                 | sddscollect -pipe=in $output -nowarning \
                 -collect=suffix=${suffix}P10  \
                 -collect=suffix=${suffix}Ave -collect=suffix=${suffix}DRange \
                 -collect=suffix=${suffix}Sigma -collect=suffix=${suffix}StDev } result] {
        return -code error "createStatsFile: $result"
    }
}


proc MakeFilterFrame {widget args} {
    set parent ""
    APSStrictParseArguments {parent}

    APSFrame $widget -parent $parent -label "Filter" 
    set w $parent$widget.frame
    global filterSignal filterLow filterHigh
    set filterSignal ""
    set filterLow 0
    set filterHigh 0
    APSLabeledEntry .le1 -parent $w -label "Signal name:" \
      -textVariable filterSignal -width 20 -contextHelp \
      "Optionally provide the name of a signal for filtering the data."
    APSLabeledEntry .le2 -parent $w -label "Lower limit:" \
      -textVariable filterLow -width 20 -contextHelp \
      "Lower limit for the filter signal."
    APSLabeledEntry .le3 -parent $w -label "Upper limit:" \
      -textVariable filterHigh -width 20 -contextHelp \
      "Upper limit for the filter signal."
}

APSApplication . -name CompareDataLogs -version $CVSRevisionAuthor -overview ""

APSScrolledStatus .status -parent .userFrame -textVariable MonDataStatus -width 60

MakeGroupSelectionFrame .group -parent .userFrame

MakeDateTimeFrame .datetime -parent .userFrame

MakeFilterFrame .filter -parent .userFrame

APSButton .cm -parent .userFrame -text "Compare means" \
  -command "CompareData -mode compareMeans"
APSButton .cuv -parent .userFrame -text "Count unlikely values" \
  -command "CompareData -mode countUnlikelyValues"
APSButton .chist -parent .userFrame -text "Compare histograms" \
  -command "CompareData -mode compareHistograms"
