#!/bin/sh  
# \
  exec oagwish "$0" "$@"

#
# $Log: not supported by cvs2svn $
# Revision 1.7  2008/02/28 18:22:56  soliday
# Fixed some bugs and added better status monitoring.
#
# Revision 1.6  2008/02/28 18:03:14  soliday
# Reindented code.
#
# Revision 1.5  2007/01/29 17:59:54  borland
# Fixed problem with filtering out of self-correlation.
#
# Revision 1.4  2006/04/24 22:30:57  jiaox
# Fixed a bug that the sddslogger only samples 100 points.
#
# Revision 1.3  2006/03/27 22:00:31  jiaox
# Fixed a typo.
#
# Revision 1.2  2006/03/24 21:27:36  jiaox
# Made sddslogger running in background mode. The previous version hung until
# sddslogger finished.
#
# Revision 1.1  2006/03/16 15:18:30  jiaox
# First version in CVS.
#
# Revision 1.1  2006/01/06 22:18:01  jiaox
# First Version in CVS.
#

set auto_path [linsert $auto_path 0  /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]
APSDebugPath
APSStandardSetup
APSDefineMonitoringVariables
package require sdds

set loggerListFilename $apsOAGDataDir/dataLoggerConfig/timeSeries.sdds

set tmpLogger [APSTmpDir]/[APSTmpString]

set CVSRevisionAuthor "\$Revision: 1.8 $ \$Author: borland $"
APSApplication . -name "Data Logger Correlation Tool" -version $CVSRevisionAuthor  

set status "Ready..."
APSScrolledStatus .status -parent .userFrame -textVariable status -width 70 -height 6 -packOption "-side top -fill x -expand false"
set standardLoggerFile ""


proc setStatus {text} {
    global status
    set status "[clock format [clock seconds] -format %H:%M:%S]: $text"
    update
}    

proc makeLoggerFrame { widget args } {
    global standardLogger loggerListFilename
    global pScroll pScrollFrame
    global Private
    
    set standardLogger ""
    set parent ""
    APSStrictParseArguments { parent }
    APSFrame $widget -parent $parent -label "Select Logger inputs"
    set w $parent$widget.frame
    APSFrame .slf -parent $w 
    APSLabeledEntry .standard -parent $w.slf.frame -label "Standard Logger:"  -textVariable  standardLogger -width 65 -packOption "-side left" \
      -contextHelp "Select logger inputs from standard loggers configured in $loggerListFilename"
    APSButton .selectStandard -parent $w.slf.frame -packOption "-side left" -text "Select..." -command selectStandardLogger \
      -contextHelp "" 		   
    APSFrame .plf -parent $w -label "Private Logger:" -packOption "-side top -fill x" 
    set w1 $w.plf.frame
    set pScroll [ APSScroll .privateScroll -parent $w1 -name "Additional Logger" -contextHelp "Add one or more additional private logger inputs." ] 
    set pScrollFrame $w1.privateScroll
    
    APSButton .add -parent $w1 -text "Add" -command "addPrivateLogger -parent $pScroll"
    APSButton .delete -parent $w1 -text "Delete" -command "deletePrivateLogger -parent $pScroll" 
    set Private(additionalLoggers) 0
    
    
    APSButton .ok -parent $w -text "  OK  " -command "generateNewLogger" -packOption "-side left" \
      -contextHelp "Must press OK to make a new logger inputs file."
    APSButton .saveAs -parent $w -text "Save As..." -command "saveLogger" -packOption "-side right"
    APSButton .edit -parent $w -text "Edit PV List" -command "editLogger" -packOption "-side right"
    
} 

set allRootNames ""
set allFileNames ""
proc firstRead {} {
    global allRootNames allFileNames loggerListFilename
    if [ string length $allRootNames ]==0 {
        if [ catch { set allRootNames [ APSGetSDDSParameter -fileName $loggerListFilename -parameter "rootname" ] }  results ] {
            setStatus "$results"
            return
        }
        
        if [ catch { set allFileNames [ APSGetSDDSParameter -fileName $loggerListFilename -parameter "Filename" ] }  results ] {
            setStatus "$results"
            return
        }
        
    }   

}

proc selectStandardLogger { } {
    global  standardLogger 
    global allRootNames allFileNames
    set tmplist ""
    firstRead
    set choiceList [ APSChooseItemFromList -name "Select Standard Logger List" \
                       -itemList $allRootNames   -returnList $allRootNames \
                       -multiItem 1 ]
    set standardLogger [ join $choiceList ,]
    
}


proc addPrivateLogger { args } {
    global Private pScrollFrame
    set parent ""
    APSStrictParseArguments { parent }
    set i $Private(additionalLoggers) 
    APSLabeledEntry .p$i -parent  $parent -width 65 -label "Additional Logger: " -textVariable Private(additionalLogger,$i) -commandButton 1
    set Private(additionalLogger,$i)  ""
    incr Private(additionalLoggers)
    tkwait visibility $parent.p$i 
    APSScrollAdjust $pScrollFrame -numVisible 2
}

proc deletePrivateLogger { args } {
    global Private
    set parent ""
    APSStrictParseArguments { parent }
    if $Private(additionalLoggers)==0 return
    incr Private(additionalLoggers) -1
    set i $Private(additionalLoggers) 
    pack forget $parent.p$i
    destroy $parent.p$i
    set Private(additionalLogger,$i) ""

}


proc generateNewLogger { } {
    global standardLogger allRootNames allFileNames
    global Private tmpLogger
    set inputList ""
    for { set i 0 } { $i < $Private(additionalLoggers) } { incr i } {
        if [llength $Private(additionalLogger,$i) ] {
            lappend inputList $Private(additionalLogger,$i)
        }	
    }     
    set tmpList [ split $standardLogger , ]
    firstRead
    foreach item $tmpList {
        set i [ lsearch $allRootNames $item ]
        if { $i<0  } {
            setStatus "Wrong standard logger rootname $item!"
            return
        }
        lappend inputList [ lindex $allFileNames $i ]	
    }   
    
    setStatus $inputList
    
    if [ llength $inputList] {
        set execute "sddscombine $inputList -merge -pipe=out \
                | sddsprocess -pipe \
                \"-redefine=column,ScaleFactor,ScaleFactor 0 == ? 1 : ScaleFactor $ \" \
                | sddssort -pipe=in -column=ReadbackName -unique $tmpLogger" 
        if [ catch { eval exec $execute } result] {
            setStatus "\nexecute\n$result"
            return
        }
        setStatus "New logger inputs stored in $tmpLogger."
    }
}

proc editLogger {} {
    global tmpLogger
    if { ![file exist $tmpLogger] } {
        setStatus "No temporary logger inputs file available!"
        return
    }   
    
    set execute "sddsedit -fileName $tmpLogger -complexAction 1 &"
    if [ catch { eval exec $execute } result] {
        setStatus "$result"
        return
    }
    setStatus "Now edit the resulting list of PVs..."    
    return
}

proc saveLogger {} {
    global tmpLogger
    if { ![file exist $tmpLogger] } {
        setStatus "No temporary logger inputs file available!"
        return
    }  
    set filename [APSFileSelectDialog .saveFile -path [pwd] -checkValidity 0  -title "Save New Logger As:"]
    if { ![llength $filename ] } {
        setStatus "Saving resulting PV list canceled."
        return
    } 
    if { [ catch {  file copy -force $tmpLogger $filename } result] } {
        setStatus $result
        return
    }
    setStatus "Saved resulting PV list to $filename."   
    
}

set Sampling(method) "points"
proc makeSamplingFrame { widget args } {
    global Sampling
    APSStrictParseArguments { parent }
    APSFrame $widget -parent $parent -label "Sampling Setup"
    set w $parent$widget.frame
    APSFrame .option -parent $w -relief flat
    set w1 $w.option.frame
    
    set ulist [ list "seconds" "minutes" "hours" "days" ]
    APSLabeledEntry .steps -parent $w1 -label "Points:" -width 10   \
      -textVariable  Sampling(npts) -packOption "-side left -expand false"
    APSLabeledEntry .rate -parent $w1 -label "Interval:" -width 10  -textVariable Sampling(rate) \
      -packOption "-side left -expand false"
    APSRadioButtonFrame .iunits -parent $w1 -label "Interval Units:" -variable Sampling(units)  \
      -buttonList {"Secs" "Mins" "Hrs" "Days"} -valueList $ulist -orientation horizontal  -packOption "-side left -expand false"
    set Sampling(units) "seconds"  
    
    
    APSFrame .output -parent $w -relief flat
    set w1 $w.output.frame
    APSLabeledEntry .output -parent $w1 -label "Output Filename:"  -textVariable  Sampling(output) -width 74 -packOption "-side left" \
      -commandButton 1	
    set Sampling(output) ""		   
    
    APSButton .start -parent $w -text "Start" -command "startLogger" -packOption "-side left"
    APSButton .status -parent $w -text "Show Status" -command "showStatus" 
}

proc startLogger {} {
    global Sampling tmpLogger
    if { [ string length $Sampling(output) ]==0 } {
        setStatus "Need a output filename!"
        return
    }
    if { ![file exist $tmpLogger] } {
        setStatus "No temporary logger inputs file available!"
        return
    }

    set execute "sddslogger $tmpLogger $Sampling(output) -sampleInterval=$Sampling(rate),$Sampling(units) -erase "
    if { $Sampling(method) == "points" } {
        if {[llength $Sampling(npts)] != 1} {
            setStatus "Missing number of sample points"
            return
        }
        if {[llength $Sampling(rate)] != 1} {
            setStatus "Missing sample interval"
            return
        }
        append execute "-steps=$Sampling(npts) "
    }
    setStatus "\n$execute"     
    if { [catch { eval exec $execute & } result ] } {
        setStatus $result
        return
    }
    setStatus "\nStart data collection"   
    after 5000 checkStatus
}

proc checkStatus {} {
    global Sampling
    if {[catch {APSGetSDDSRowCount -fileName $Sampling(output)} row]} {
        setStatus "Error: $row"
        return
    }
    if {$row < $Sampling(npts)} {
        setStatus "[expr {$row * 100.0 / $Sampling(npts)}]% done"
        after 5000 checkStatus 
    } else {
        setStatus "Data collection done."
    }
}

proc showStatus {} {
    global Sampling
    if { [ string length $Sampling(output) ]==0 } {
        setStatus "Need a output filename!"
        return
    }
    if { [catch {APSGetSDDSRowCount -fileName $Sampling(output) } row] } {
        setStatus "$row"
	return
    }
    
    if { $row < $Sampling(npts) } {
        setStatus "Performing data collection...\nNPTS: $row"
    } else {
        setStatus "Data collection done."
    }       	
    
}
proc makeCorrelateFrame { widget args } {
    global Correlate Sampling
    APSStrictParseArguments { parent}
    set Correlate(xVar) ""
    set Correlate(rankOrder) "-rankOrder"
    set Correlate(stDevOutlier) 0
    set Correlate(outlierLimit) 1
    set Correlate(outlierPasses) 1
    set Correlate(output) ""
    set Correlate(make2DHistograms) 0
    set Correlate(ditherEnable) 1
    set Correlate(ditherAmount) 0.005
    set Correlate(plotLimit) 0.5
    set Correlate(plotLayout) ""
    APSFrame $widget -parent $parent -label "Correlate" 
    set w $parent$widget.frame
    APSFrame .f1 -parent $w -relief flat
    set w1 $w.f1.frame
    APSLabeledEntry .var -parent $w1 -label "Independent PV: " -textVariable Correlate(xVar) -width 65 -packOption "-side left" 
    APSButton .varSelect -parent $w1 -text "Select..." -command selectVar -packOption "-side right"
    
    APSFrame .f2 -parent $w -relief flat
    set w1 $w.f2.frame
    APSRadioButtonFrame .rankOrder -parent $w1 -orientation horizontal -variable Correlate(rankOrder) -label "Rank-order: " \
      -buttonList "Yes NO" -valueList { "-rankOrder" "" }  -packOption "-side left" -contextHelp \
      "Choose whether to use rank-order correlations or standard correlations of values. The former is considered more robust."
    APSRadioButtonFrame .outlier -parent $w1 -orientation horizontal -variable Correlate(stDevOutlier) -label "Remove outliers: " \
      -buttonList "Yes NO" -valueList "1 0"   -packOption "-side left -expand 0" \
      -commandList [ list "APSEnableWidget $w1.olimit;APSEnableWidget $w1.passes" \
                       "APSDisableWidget $w1.olimit;APSDisableWidget $w1.passes"] \
      -contextHelp "Choose whether to remove outliers prior to correlation analysis."		       
    
    
    APSLabeledEntry .olimit -parent $w1 -label "St.dev.limit:" -textVariable Correlate(outlierLimit) -width 5 -type real \
      -packOption "-side left -expand 0"
    APSLabeledEntry .passes -parent $w1 -label "Passes:" -textVariable Correlate(outlierPasses) -width 5 -type real 
    APSDisableWidget $w1.olimit;APSDisableWidget $w1.passes
    
    APSFrame .f3 -parent $w -relief flat 
    set w1 $w.f3.frame
    APSLabeledEntry .dither -parent $w1 -label "Dither amount for scatter plots:" -textVariable Correlate(ditherAmount) \
      -enableVariable Correlate(ditherEnable) -width 10 -type real -packOption "-side left -expand 0"  -contextHelp \
      "Choose whether to dither the values for scatter plots in order to get a better idea of the density of points when points are closely spaced (as when you are near the digitizer resolution limit).  This is a faster alternative to using 2D histograms.  The value is a fraction of the range of the data."
    APSLabeledEntry .limit -parent $w1 -label "Correlation limit for scatter plots:" -textVariable Correlate(plotLimit) -width 10 -type real \
      -contextHelp "Enter the minimum correlation for which plots will be made."
    
    APSRadioButtonFrame .layout -parent $w -label "Plot layout: " \
      -orientation horizontal -variable Correlate(plotLayout) \
      -buttonList "1x1 1x2 2x1 2x2 3x3 4x4 5x5 6x6" \
      -valueList  [list "" -layout=1,2 -layout=2,1 -layout=2,2 -layout=3,3 -layout=4,4 -layout=5,5 -layout=6,6 ] \
      -contextHelp \
      "Selected the plot layout, which gives the number of plots horizontally and vertically on each panel."
    APSRadioButtonFrame .hist2d -parent $w -label "Make 2D Histograms:" \
      -orientation horizontal -variable Correlate(make2DHistograms) \
      -buttonList "Yes No" -valueList "1 0" -contextHelp \
      "Choose whether to make and plot 2d histograms.  Takes extra time."
    
    APSButton .run -parent $w -text "Analyze"      -command "RunCorrelate"
    APSButton .display -parent $w -text "ReDisplay " -command "DisplayCorrelate"
    APSButton .save -parent $w -text "Save Result As..." -command "SaveCorrelate" -packOption "-side right"
    
}

proc selectVar { } {
    global Sampling Correlate
    
    if { ![file exist $Sampling(output) ] } {
        setStatus "No data file available!"
        return
    }    
    
    set varList ""
    if [ catch {APSGetSDDSNames -class column  -fileName $Sampling(output) } varList ] {
        setStatus "$varList"
        return
    }
    
    set Correlate(xVar) [APSChooseItemFromList -name "independent variable" \
                           -itemList $varList -returnList $varList -multiItem 0]
}


proc RunCorrelate {} {
    global Correlate Sampling
    if ![string length $Sampling(output) ] {
        setStatus "No data file. Please provide a output filename for the sampling."
        return
    }
    if { ![file exist $Sampling(output)] } {
        setStatus "No data file. Please start data collection first."
        return
    }
    if ![string length $Correlate(xVar)] {
        setStatus "Choose one independent variable"
        return
    }   
    
    set Correlate(output) "/tmp/[APSTmpString]"
    
    set command "sddscorrelate $Sampling(output) $Correlate(rankOrder) -withOnly=$Correlate(xVar) "
    if $Correlate(stDevOutlier) {
        append command "-stDevOutlier=limit=$Correlate(outlierLimit),passes=$Correlate(outlierPasses) "
    }
    
    append command " -pipe=out | sddsprocess -pipe \"-define=column,AbsCC,CorrelationCoefficient abs\" -match=col,Correlate1Name=@Correlate2Name,! | sddssort -pipe=in $Correlate(output) -column=AbsCC,decreasing "
    
    setStatus "$command"
    if [catch {eval exec $command} result ] {
        setStatus "$result"
        return
    }
    setStatus "Correlation calculation done."
    DisplayCorrelate      

}

proc DisplayCorrelate { } {
    global Correlate Sampling
    set tmpFile [APSTmpDir]/[APSTmpString]
    exec sddsprintout $Correlate(output) $tmpFile \
      -column=CorrelationCoefficient,format=%12.3f,label=Corr.Coef. \
      -column=CorrelationSignificance,format=%12.3f,label=Corr.Signif. \
      -column=CorrelatePair,format=%60s -width=110
    APSFileDisplayWindow [APSUniqueName .] -width 120 -height 20 \
      -fileName $tmpFile -deleteOnClose 1 -sddsExportableFile $Correlate(output) \
      -contextHelp "Shows the results of correlation analysis for $Sampling(output).  The correlation significance is the probability of the observed correlation happening by chance when the variables are actually uncorrelated.  Hence, a smaller value means the correlation is more likely to be real."
    
    if [ catch { sdds load $Correlate(output) data }  result ] {
        return -code error "$result"
    }
    set rows [llength [lindex $data(Column.Correlate1Name) 0]]
    set correlate1List [lindex $data(Column.Correlate1Name) 0]
    set correlate2List [lindex $data(Column.Correlate2Name) 0]
    set coefList [lindex $data(Column.CorrelationCoefficient) 0]
    set sigList [lindex $data(Column.CorrelationSignificance) 0]
    set plotOptionList ""
    file delete $Correlate(output).h2d	   
    set index 0
    foreach name1 $correlate1List name2 $correlate2List coef $coefList sig $sigList {
        if [expr abs($coef)<$Correlate(plotLimit)] continue
        set coef [format %10.4g $coef]
        set sig [format %10.4g $sig]
        lappend plotOptionList -column=$name1,$name2 
        lappend plotOptionList "-title=Corr. Coef.: $coef    Signif.: $sig"
        if $Correlate(make2DHistograms) {
	    exec sddshist2d $Sampling(output) $Correlate(output).h2d.$index \
              -column=$name1,$name2 -xparam=$rows -yparam=$rows -minimumScale=1e-12
            lappend h2dList $Correlate(output).h2d.$index
            incr index
        }
    }
    if ![llength $plotOptionList] {
        setStatus "No correlations greater than $Correlate(plotLimit) found"
        return
    }
    set dither [expr $Correlate(ditherAmount)*$Correlate(ditherEnable)]
    
    eval exec sddsplot $Sampling(output) -dither=xrange=$dither,yrange=$dither \
      -separate -graph=dot $plotOptionList $Correlate(plotLayout) &
    if $Correlate(make2DHistograms) {
        eval exec sddscombine $h2dList $Correlate(output).h2d -overWrite
        exec sddscontour -shade $Correlate(output).h2d &
        eval file delete -force $h2dList 
    }
}


proc SaveCorrelate {} {
    global Correlate 
    
    if ![string length $Correlate(output)] {
        setStatus "Nothing to save. Analyze first."
        return
    }   
    if { ![file exist $Correlate(output)] } {
        setStatus "No result available!"
        return
    }  
    set filename [APSFileSelectDialog .saveResults -path [pwd] -checkValidity 0 -title "Save Correlation Results As:" ]
    if { ![llength $filename ] } {
        setStatus "Saving correlation results canceled."
        return
    } 
    if { [ catch {  file copy -force $Correlate(output) $filename } result] } {
        setStatus $result
        return
    }
    setStatus "Saved correlation results to $filename."   
    
    
}

makeLoggerFrame .logger -parent .userFrame
makeSamplingFrame .sampling -parent .userFrame
makeCorrelateFrame .correlate -parent .userFrame

