#!/bin/sh
# \
exec oagwish "$0" "$@"


set debugReviewAlarmLog 0

set auto_path [linsert $auto_path 0  /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0  /usr/local/oag/lib_patch/$env(HOST_ARCH)]
APSDebugPath

set CVSRevisionAuthor "\$Revision: 1.8 $ \$Author: soliday $"

proc SetStatusText {text} {
    global statusText
    set statusText "[exec date +%H:%M:%S]: $text"
    update
}

proc MakeDataSelectionWidget {widget args} {
    set rootname ""
    set parent ""
    set label ""
    APSStrictParseArguments {rootname parent label}

    APSFrame $widget -parent $parent -label $label  \
      -contextHelp "Enter the date and time range of interest.  You may use HH:MM or HH:MM:SS syntax for times, or just enter time as decimal hours."
    set w $parent$widget.frame
    global ${rootname}TimeStart ${rootname}TimeEnd ${rootname}Year ${rootname}Day ${rootname}Month
    set ${rootname}TimeStart 0
    set ${rootname}TimeEnd 24
    APSLabeledEntryFrame .time -parent $w -label "Time (start, end): " \
      -variableList "${rootname}TimeStart ${rootname}TimeEnd" \
      -orientation horizontal -width 8
    APSDateTimeAdjEntry .date -parent $w -dayVariable ${rootname}Day \
      -monthVariable ${rootname}Month -yearVariable ${rootname}Year \
      -label "Date: " -buttonSize small
    MakeDataFilteringWidget .filter -parent $w -rootname $rootname
}

proc MakeDataFilteringWidget {widget args} {
    set parent ""
    set rootname ""
    APSStrictParseArguments {parent rootname}

    global ${rootname}PvFilterFrame
    APSFrame $widget -parent $parent -label "Data filtering" 
    set w $parent$widget.frame 
    set ${rootname}PvFilterFrame $w
    APSLabeledEntry .pvMatch -parent $w \
      -label "PV name match: " -width 40 -textVariable ${rootname}PvMatch \
      -contextHelp "Enter a wildcard string with which to select PV names. \
* matches 0 or more characters, while ? matches one character. \
 \[<text>\] matches one occurence of any character in <text>. \
 For example, all available storage ring A:Q1 and B:Q1 could be selected with S*\[AB\]:Q1*. "
    APSLabeledEntry .pvExclude -parent $w \
      -label "PV name exclude: " -width 40 -textVariable ${rootname}PvExclude \
      -contextHelp "Enter a wildcard string with which to exclude PV names. \
Use a blank entry to exclude none. \
* matches 0 or more characters, while ? matches one character. \
 \[<text>\] matches one occurence of any character in <text>. \
 For example, all available storage ring A:Q1 and B:Q1 could be selected with S*\[AB\]:Q1*. "

}

proc MakeAnalysisControls {widget args} {
    set parent ""
    APSStrictParseArguments {parent}

    APSFrame $widget -parent $parent -label "Analysis Controls"
    set w $parent$widget.frame
    APSCheckButtonFrame .severityCB -parent $w -label "Severity: " \
      -buttonList {MAJOR MINOR INVALID NO_ALARM} -variableList {doMajor doMinor doInvalid doNoAlarm} \
      -contextHelp "Choose the severity levels for which you wish to see data." \
      -orientation horizontal  -allNone 1

    APSLabeledEntry .timeBin -parent $w -textVariable timeBin \
      -width 10 -label "Time bin (min): " -contextHelp \
      "Enter the width in minutes of time bins for the analysis.  Using a smaller time bin usually gives less reliable analysis."
    APSLabeledEntry .threshold -parent $w -label "Significance threshold (percent): " \
      -textVariable significanceThreshold -width 10 -contextHelp \
      "Enter the significance threshold for the probability printout.  A lower value means that the probability of a real difference must be greater in order for the data to be printed."
    APSCheckButtonFrame .cbs -parent $w -label "Analyses: " -buttonList {Printout "Total Time Histogram" "Individual Time Histograms"} \
      -variableList {printout totalHistogram individualHistograms } -orientation horizontal \
      -contextHelp "Selecting Printout gives a printout showing counts and significance levels for each channel.  Selecting Total Time Histogram gives a histogram vs. time for all channels together.  Selecting Individual Time Histograms does the same, but for each channel separately."

    APSButton .run -parent $w -text "Run analysis" \
      -command {RunAlarmProbabilityAnalysis -major $doMajor -minor $doMinor \
                  -invalid $doInvalid -noAlarm $doNoAlarm \
                  -individualHistograms $individualHistograms -totalHistogram $totalHistogram \
                  -printout $printout -significanceThreshold $significanceThreshold \
                  -timeBin $timeBin \
                  -systemChoice $systemChoice \
                  -refPvMatch $refPvMatch -refPvExclude $refPvExclude \
                  -dataPvMatch $dataPvMatch -dataPvExclude $dataPvExclude \
                  -refFilterChoices $refFilterChoices \
                  -dataFilterChoices $dataFilterChoices \
                  -refDateList [APSFormatDate -year $refYear -day $refDay -month $refMonth -dateFormat list] \
                  -dataDateList [APSFormatDate -year $dataYear -day $dataDay -month $dataMonth -dateFormat list] \
                  -refTimeStart $refTimeStart -refTimeEnd $refTimeEnd \
                  -dataTimeStart $dataTimeStart -dataTimeEnd $dataTimeEnd } \
      -contextHelp "Runs the analysis using the parameters set above."
}

proc RunAlarmProbabilityAnalysis {args} {
    global SystemRootnames Systems dataDir filterFileList filterDescriptionList
    set major 0
    set minor 0
    set invalid 0
    set noAlarm 0
    set individualHistograms  0
    set totalHistogram 1
    set printout 1
    set refPvMatch *
    set refPvExclude ""
    set dataPvMatch *
    set dataPvExclude ""
    set timeBin 0
    set systemChoice ""
    set refDateList 0
    set dataDateList 0
    set refTimeStart 0
    set dataTimeStart 0
    set refTimeEnd 0
    set dataTimeEnd 0
    set refFilterChoices ""
    set dataFilterChoices ""
    set significanceThreshold 100
    APSStrictParseArguments {major minor invalid noAlarm individualHistograms totalHistogram printout \
                               timeBin systemChoice refDateList dataDateList \
                               refTimeStart dataTimeStart refTimeEnd dataTimeEnd \
                               significanceThreshold refPvMatch refPvExclude dataPvMatch dataPvExclude \
                               refFilterChoices dataFilterChoices}

    set systemRootname [lindex $SystemRootnames [lsearch -exact $Systems $systemChoice ]]
    foreach root {ref data} {
        set ${root}Year  [lindex [subst \$${root}DateList] 0]
        set ${root}Month [lindex [subst \$${root}DateList] 2]
        set ${root}Day   [lindex [subst \$${root}DateList] 3]
        foreach suffix {TimeStart TimeEnd} {
            if [catch {APSConvertTimeToHours [subst \$${root}${suffix}]} ${root}${suffix}H] {
                SetStatusText "Bad time value: [subst \$${root}${suffix}]"
                return
            }
            if [subst \$${root}${suffix}H]<0 {
                set ${root}${suffix}H 0
                set ${root}${suffix} 0
            }
            if [subst \$${root}${suffix}H]>24 {
                set ${root}${suffix}H 24
                set ${root}${suffix} 24
            }
        }
        if [subst \$${root}TimeStartH]>[subst \$${root}TimeEndH] {
            SetStatusText "Error: Start time later than end time."
            return
        }
    }
    
    foreach root {ref data} {
        set ${root}File [APSFindFilesBetweenDates -rootname ${systemRootname}- \
                         -directory $dataDir/$systemRootname \
                           -startDateList [subst \$${root}DateList] \
                           -endDateList [subst \$${root}DateList]]
        if [llength [subst \$${root}File]]==0 {
            SetStatusText "Error: No files for [APSFormatDate -dateList [subst \$${root}DateList] -dateFormat Y-M-D]"
            return
        }
        SetStatusText "[llength ${root}File] files for [APSFormatDate -dateList [subst \$${root}DateList] -dateFormat Y-M-D]"
        if [llength [subst \$${root}File]]>1 {
            SetStatusText "Merging files..."
            set file /tmp/[APSTmpString]
            eval exec sddscombine [subst \$${root}File] -merge $file
            set ${root}File $file
            APSAddToTempFileList $file
        }
        set tailroot [file rootname [file tail [subst \$${root}File]]]
        set derefFile /tmp/$tailroot.deref.$root
        SetStatusText "Dereferencing $root file...."
        APSAddToTempFileList $derefFile
        if [catch {DereferenceAlarmLoggerData -fileName [subst \$${root}File] \
                     -timeMin [subst \$${root}TimeStartH] -timeMax [subst \$${root}TimeEndH] \
                     -timeName TimeOfDay \
                     -newFile $derefFile} result ] {
            SetStatusText "Error: $result"
            return
        }

        set filterList {}
        set pvMatch [subst \$${root}PvMatch]
        if [string compare "$pvMatch" "*"] {
            lappend filterList [MakeMatchOption -pvMatch $pvMatch]
        }

        set pvExclude [subst \$${root}PvExclude]
        if [string length "$pvExclude"] {
            lappend filterList [MakeMatchOption -pvExclude $pvExclude]
        }

        SetStatusText "Filter file choices: [join [subst \$${root}FilterChoices] \n]"
        if [llength [subst \$${root}FilterChoices] ] {
            set filterFiles [MakeFilterFileSelectionList \
                               -files $filterFileList \
                               -descriptions $filterDescriptionList \
                               -choices [subst \$${root}FilterChoices] ]
            set filterFile /tmp/[APSTmpString]
            set newFile /tmp/[APSTmpString]
            APSAddToTempFileList $filterFile $newFile
            SetStatusText "Filtering using filter files..."
            if {[catch {eval exec sddscombine $filterFiles $filterFile -merge} result] || \
                  [catch {eval exec sddsselect $derefFile $filterFile $newFile \
                            -match=ControlName -reuse=page,row} result]} {
                SetStatusText "Error: $result"
                return 
            }
            set derefFile $newFile
        }
                         
        if {$major || $minor || $invalid || $noAlarm} {
            set no_alarm $noAlarm
            set qualifierList ""
            foreach severity {major minor invalid no_alarm} {
                if [subst \$$severity] {
                    lappend qualifierList AlarmSeverity=[string toupper $severity]
                }
            }
            switch [llength $qualifierList] {
                0 {
                }
                1 {
                    lappend filterList "-match=column,[lindex $qualifierList 0]"
                }
                default {
                    lappend filterList \
                      "-match=column,[lindex $qualifierList 0],[join [lrange $qualifierList 1 end] ,|,],|"
                }
            }
        }
        if [string length $filterList] {
            SetStatusText "Filtering $root file..."
            set newFile $derefFile.filt 
            APSAddToTempFileList $newFile
            if [catch {eval exec sddsprocess $derefFile $newFile $filterList} result] {
                SetStatusText "Error: $result"
                return
            }
            set derefFile $newFile
        }
        set ${root}File $derefFile
    }
    if [catch {exec sddsprocess $refFile -pipe=out -define=param,Count,n_rows,type=long \
                 | sdds2stream -pipe -param=Count} refCount] {
        SetStatusText "Error: $refCount"
    }
    if !$refCount {
        SetStatusText "No counts for reference date/time---can't do analysis."
    }
    set refRate [expr (1.0*$refCount)/($refTimeEndH-$refTimeStartH)]
    SetStatusText "Reference rate: $refRate/h"

    if [catch {exec sddsprocess $dataFile -pipe=out -define=param,Count,n_rows,type=long \
                 | sdds2stream -pipe -param=Count} dataCount] {
        SetStatusText "Error: $dataCount"
    }
    SetStatusText "Data count: $dataCount"
    set expectedCount [expr $refRate*($dataTimeEndH-$dataTimeStartH)]
    SetStatusText "Expected:   [format %.3g $expectedCount]"
    set meanCount [expr ($expectedCount+$dataCount)/2.0]
    if $expectedCount==$dataCount {
        set prob 1
    } elseif $expectedCount<$dataCount {
        set prob [exec rpnl 1 $expectedCount $meanCount poiSL - $dataCount $meanCount poiSL +]
    } else {
        set prob [exec rpnl 1 $dataCount $meanCount poiSL - $expectedCount $meanCount poiSL +]

    }
    SetStatusText "Probability [format %.2f [expr $prob*100]]% of no difference in total alarm rates."

    
    if $printout {
        SetStatusText "Sorting and cross-referencing..."
        if {[catch {exec sddssort $refFile -pipe=out -column=ControlNameIndex -unique=count \
                      | sddsprocess -pipe=in $refFile.sort \
                      "-define=param,TimeFactor,$dataTimeEndH $dataTimeStartH - $refTimeEndH $refTimeStartH - /" \
                      {-define=column,ExpectedCount,IdenticalCount TimeFactor *}} result] || \
              [catch {exec sddssort $dataFile -pipe=out -column=ControlNameIndex -unique=count \
                        | sddsxref -nowarning -pipe -match=ControlName $refFile.sort -fillIn \
                        | sddsprocess -pipe -nowarning \
                        {-define=column,MeanCount,IdenticalCount ExpectedCount + 2 /} \
                        {-define=column,MinCount,IdenticalCount ExpectedCount 2 minn} \
                        {-define=column,MaxCount,IdenticalCount ExpectedCount 2 maxn} \
                        {-define=column,ProbNoRealDiff,1 MinCount 1 + MeanCount poiSL - MaxCount MeanCount poiSL + 100 * 100 2 minn,units=%} \
                        -filter=column,ProbNoRealDiff,0,$significanceThreshold \
                        | sddssort -pipe=in $dataFile.result -column=ProbNoRealDiff,incr -column=ControlName } result]} {
            SetStatusText "Error: $result"
            return
        }
        SetStatusText "Making printout..."
        set tmpFile /tmp/[APSTmpString]
        if [catch {exec sddsprintout $dataFile.result $tmpFile \
                     "-title=$systemChoice alarm data on [APSFormatDate -dateList $dataDateList -dateFormat Y-M-D] \[$dataTimeStart\\\, $dataTimeEnd\]h compared to reference data on [APSFormatDate -dateList $refDateList -dateFormat Y-M-D] \[$refTimeStart\\\, $refTimeEnd\]h" \
                     -column=ControlName,format=%38s \
                     -column=AlarmSeverity -column=AlarmStatus \
                     "-column=IdenticalCount,label=Actual Count" \
                     "-column=ExpectedCount,label=Expected Count,format=%12.4f" \
                     "-column=ProbNoRealDiff,format=%6.2f"} result] {
            SetStatusText "Error: $result"
            return
        }
        APSFileDisplayWindow [APSUniqueName .] -fileName $tmpFile -deleteOnClose 1 -width 126 \
          -height 40
    }

    if $totalHistogram {
        SetStatusText "Making histogram for all channels taken together..."
        set expectedCount [expr $refRate*($timeBin/60.0)]
        set tmpFile /tmp/[APSTmpString]
        APSAddToTempFileList $tmpFile
        if [catch {exec sddshist $dataFile -pipe=out -data=TimeOfDay -sizeOfBins=[expr $timeBin/60.0] \
                     -lowerLimit=$dataTimeStartH -upperLimit=$dataTimeEndH \
                     | sddsprocess -pipe=in $tmpFile \
                     "-define=column,ProbNoRealIncrease,frequency $expectedCount poiSL 100 *,units=%"} result] {
            SetStatusText "Error: $result"
        }
        SetStatusText "Plotting histogram."
        exec sddsplot -groupby=namestring -sep=namestring $tmpFile \
          -layout=1,2 -labelsize=0.03 -col=TimeOfDay,ProbNoRealIncrease -ticks=ygrid \
          -column=TimeOfDay,frequency  -graph=line \
          -column=TimeOfDay,frequency -graph=sym,subtype=1 -filter=column,ProbNoRealIncrease,0,5 $tmpFile &
    }

    if $individualHistograms {
        SetStatusText "Sorting for individual histograms..."
        APSAddToTempFileList $refFile.sort $dataFile.sort
        if {[catch {exec sddssort $refFile $refFile.sort -column=ControlName -unique=count} result] || \
              [catch {exec sddssort $dataFile -pipe=out -column=ControlName -unique \
                        | sddsxref -pipe=in $refFile.sort $dataFile.sort -fillIn -nowarning -match=ControlName} result]} {
            SetStatusText "Error: $result"
            return
        }
        if [catch {APSGetSDDSColumn -fileName $dataFile.sort -column ControlName} pvNameList] {
            SetStatusText "Error: $pvNameList"
            return
        }
        if [catch {APSGetSDDSColumn -fileName $dataFile.sort -column IdenticalCount} refCountList] {
            SetStatusText "Error: $refCountList"
            return
        }
        set tmpData /tmp/[APSTmpString]
        set index 0
        set histFileList ""
        foreach pvName $pvNameList {
            set refRows [lindex $refCountList $index]
            SetStatusText "Working on $pvName..."
            APSAddToTempFileList $dataFile.$pvName.his
            set expectedCount [expr ($refRows?$refRows:1)/($refTimeEndH-$refTimeStartH)*($timeBin/60.0)]
            if [catch {exec sddsprocess $dataFile -pipe=out -match=col,ControlName=$pvName \
                         | sddshist -pipe  -data=TimeOfDay -sizeOfBins=[expr $timeBin/60.0] \
                         -lowerLimit=$dataTimeStartH -upperLimit=$dataTimeEndH \
                         | sddsprocess -pipe=in $dataFile.$pvName.his \
                         -print=param,pvName,$pvName \
                         "-define=column,ProbNotUnusualRate,frequency $expectedCount poiSL 100 *,units=%"} result] {
                SetStatusText "Error: $result"
                return
            }
            lappend histFileList $dataFile.$pvName.his
            incr index
        }
        SetStatusText "Plotting histograms."
        eval exec sddsplot -labelsize=0.03 -layout=1,2 -groupby=fileindex,namestring -sep=namestring \
          $histFileList -column=TimeOfDay,frequency -graph=line \
          -column=TimeOfDay,ProbNotUnusualRate -ticks=ygrid -title=@pvName \
          -column=TimeOfDay,frequency -graph=sym,subtype=1 -filter=column,ProbNotUnusualRate,0,5 &
    }
}

proc SwitchSystem {} {
    global systemChoice dataDir filterFileList filterDescriptionList
    SetStatusText "Switched to $systemChoice"

    foreach type {ref data} {
        foreach var {FilterFileWidget FilterChoices PvFilterFrame} {
            global $type$var
            set $var [subst \$$type$var]
        }
        set ${type}FilterChoices {}
    }

    set filterIndexFile $dataDir/FilterFiles/$systemChoice.index
    set filterDescriptionList {}
    set filterFileList {}
    if [file exists $filterIndexFile] {
        if {[catch {APSGetSDDSColumn -fileName $filterIndexFile -column FilterDescription} filterDescriptionList] || \
              [catch {APSGetSDDSColumn -fileName $filterIndexFile -column FilterFile} filterFileList]} {
            APSSetVarAndUpdate statusText "Problem reading filter index file for $systemChoice."
            return
        }
    }

    foreach type {ref data} {
        foreach var {FilterFileWidget PvFilterFrame} {
            set $var [subst \$$type$var]
        }
        if ![llength $filterDescriptionList] {
            if {[string length $FilterFileWidget] && [winfo exists $FilterFileWidget]} {
                destroy $FilterFileWidget
            }
            continue
        }

        if {[string length $FilterFileWidget] && [winfo exists $FilterFileWidget]} {
            $FilterFileWidget.userFrame.sl.listbox delete 0 end
            eval $FilterFileWidget.userFrame.sl.listbox insert 0 $filterDescriptionList
        } else {
            APSScrolledListWindow .fflist -parent $PvFilterFrame -height 3 \
              -selectionVar ${type}FilterChoices -label "Filter choices"  -autoAccept 1 \
              -closeButton 0 -acceptButton 0 -clearButton 1
            set FilterFileWidget $PvFilterFrame.fflist
            eval $FilterFileWidget.userFrame.sl.listbox insert 0 $filterDescriptionList
        }
        foreach var {FilterFileWidget PvFilterFrame} {
            set $type$var [subst \$$var]
        }
    }
}

proc MakeFilterFileSelectionList {args} {
    set files {}
    set descriptions {}
    set choices {} 
    APSStrictParseArguments {files descriptions choices}
    set selectionList {}
    global dataDir
    foreach elem $choices {
        set index [lsearch -exact $descriptions $elem]
        if $index!=-1 {
            lappend selectionList $dataDir/FilterFiles/[lindex $files $index]
        } else {
            SetStatusText "Not found: $elem"
        }
    }
    return $selectionList
}

proc SetNoncustomMode {} {
    global userFilename pickFileWidget filterWidget filterFileWidget filterFileList filterDescriptionList filterChoices
    global directoryWidget filenameWidget  defaultDataDir pvFilterFrame systemChoice dataDir
    set userFilename ""
#    APSDisableButton $pickFileWidget
    global Systems SystemRootnames IsAlarmLogger SystemIndex
    set index [lsearch $Systems $systemChoice]
    set SystemIndex $index
    set dataDir $defaultDataDir/[lindex $SystemRootnames $index]
#    $filterWidget configure -state disabled
#    $directoryWidget configure -state disabled
#    $filenameWidget configure -state disabled

    set filterIndexFile $defaultDataDir/FilterFiles/$systemChoice.index
    set filterDescriptionList {}
    set filterFileList {}
    set filterChoices {}
    if [file exists $filterIndexFile] {
        if {[catch {APSGetSDDSColumn -fileName $filterIndexFile -column FilterDescription} filterDescriptionList] || \
              [catch {APSGetSDDSColumn -fileName $filterIndexFile -column FilterFile} filterFileList]} {
            APSSetVarAndUpdate statusText "Problem reading filter index file for $systemChoice."
            return
        }
    }
}

APSApplication . -name AlarmProbabilityAnalysis -version $CVSRevisionAuthor \
  -overview {This tool does analysis of alarm data to assess the probability of alarm densities.}

AlarmReviewSetup -customSystem 0

set doMajor   1
set doMinor   0
set doInvalid 0
set doNoAlarm 0
set individualHistograms 0
set printout 1
set totalHistogram 1
set refPvMatch *
set refPvExclude ""
set dataPvMatch *
set dataPvExclude ""
set timeBin 10
set significanceThreshold 100
set dataDir /home/helios/oagData/Alarms
set filterFileList ""
set filterDescriptionList ""
foreach type {data ref} {
    foreach var {FilterFileWidget FilterChoices PvFilterFrame} {
        set $type$var ""
    }
}

set statusText Ready.
APSScrolledStatus .status -parent .userFrame -width 80 -textVariable statusText

APSRadioButtonFrame .systemRB -parent .userFrame -label "System: " \
  -buttonList $Systems -valueList $Systems -variable systemChoice \
  -contextHelp "Choose the system for which you wish to see data." \
  -orientation horizontal \
  -commandList [APSReplicateItem -item SwitchSystem -number [llength $Systems]]

MakeDataSelectionWidget .timeRef -parent .userFrame \
  -rootname ref -label "Reference data"
MakeDataSelectionWidget .timeAnal -parent .userFrame \
  -rootname data -label "Analysis data"
MakeAnalysisControls .analysisControls -parent .userFrame

dp_atexit append {SetStatusText "Cleaning up prior to exit..."}
