#!/bin/sh  
# \
  exec oagwish "$0" "$@"

set auto_path [linsert $auto_path 0 /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]

set apsttk 1
set CVSRevisionAuthor "\$Revision: 1.36 $ \$Author: soliday $"

APSApplication . -name checkDataLoggers -version $CVSRevisionAuthor 

set status Ready.
APSScrolledStatus .status -parent .userFrame -textVariable status \
  -width 115 -withButtons 1 -height 30 -packOption "-fill both -expand true"
set expertMode 1
set verboseMode 0
APSRadioButtonFrame .verbose -parent .userFrame -variable verboseMode -label "Verbose mode: " \
  -buttonList {Yes No} -valueList {1 0} -orientation horizontal \
  -contextHelp "Verbose mode provides more output as the checks proceed."
APSButton .check -parent .userFrame -text Check -command "CheckLoggerFiles ; CheckRunControls" -width ""
APSButton .log -parent .userFrame -text "View Log" -command "ViewLog" -width ""
APSButton .checkRC -parent .userFrame -text "Check RunControls" -command "CheckRunControls" -width ""
APSButton .listRC -parent .userFrame -text "List RunControl PVs" -command "CheckRunControls -list 1" -width ""
APSButton .intervals -parent .userFrame -text "Check Interval Consistancy" -command "CheckIntervalConsistancy" -width ""


proc CheckIntervalConsistancy {args} {
    APSSetVarAndUpdate status "Checking data strobe loggers on gladiator for logging consistancy for the last 60 minutes.\nThese loggers will occationally not log on the strobe signal due to IOC issues\nThe unconditional loggers should be 99-100%. If it is below that, please email soliday@anl.gov"
    foreach rate "1a 1b 1c 1d 2 4 8 16 32 64a 64b 128 256" {
        set f /home/helios/oagData/logging/inputFiles/unconditional.${rate}.sdds
        if {![file exists $f]} {
            continue
        }
        if {[catch {exec sdds2stream $f -rows=bare} pvcount]} {
            APSSetVarAndUpdate status $pvcount
            return
        }
        if {[catch {exec sddsprocess $f -pipe=out -clip=1,0,invert | sdds2stream -pipe=in -col=ReadbackName} rbname]} {
            APSSetVarAndUpdate status $rbname
            return
        }
        set r $rate
        if {$rate == "1a"} {
            set r 1
        }
        if {$rate == "1b"} {
            set r 1
        }
        if {$rate == "1c"} {
            set r 1
        }
        if {$rate == "1d"} {
            set r 1
        }
        if {$rate == "64a"} {
            set r 64
        }
        if {$rate == "64b"} {
            set r 64
        }
        set currentFile [lindex [exec ls -1tr /home/helios/oagData/logging/${r}/$rbname] end]
        set currentFile [file join /home/helios/oagData/logging/${r}/${rbname} $currentFile]
        set sec [expr [clock seconds] - (60 * 60)]

        if {[catch {exec sddsprocess $currentFile -pipe=out -filter=column,Time,$sec,[clock seconds] | sdds2stream -pipe=in -col=Time} tsList]} {
            APSSetVarAndUpdate status $tsList
            return
        }
        set missed 0
        set t [lindex $tsList 0]
        set i 0
        set ts [lindex $tsList $i]
        while {$t < [lindex $tsList end]} {
            if {[expr (round($t))] != [expr (round($ts))]} {
                incr missed
                set t [expr $t + $r]
                continue
            }
            incr i
            set ts [lindex $tsList $i]
            set t [expr $t + $r]
        }
        set count [llength $tsList]
        if {$count == 0} {
            set count 1
            set missed 1
        }
        APSSetVarAndUpdate status "unconditional.${rate} [format %.2f [expr 100.0 * ($count) / ($count + $missed)]]% [format %6d $pvcount] PVs"
    }
    foreach rate "1 2 4 8 16 32 64 128 256" {
        foreach num "1 2 3 4 5 6 7 8" {
            foreach letter "a b" {
                set f /home/helios/oagData/logging/inputFiles/conditional${num}.${rate}${letter}.sdds
                if {![file exists $f]} {
                    if {$letter == "b"} {
                        continue
                    }
                    set f /home/helios/oagData/logging/inputFiles/conditional${num}.${rate}.sdds
                    set letter ""
                    if {![file exists $f]} {
                        continue
                    }
                }
                if {[catch {exec sdds2stream $f -rows=bare} pvcount]} {
                    APSSetVarAndUpdate status $pvcount
                    return
                }
                if {[catch {exec sdds2stream $f -para=Condition} condition]} {
                    APSSetVarAndUpdate status $condition
                    return
                }
                if {[catch {exec sddsprocess $f -pipe=out -clip=1,0,invert | sdds2stream -pipe=in -col=ReadbackName} rbname]} {
                    APSSetVarAndUpdate status $rbname
                    return
                }
                set currentFile [lindex [exec ls -1tr /home/helios/oagData/logging/${rate}/$rbname] end]
                set currentFile [file join /home/helios/oagData/logging/${rate}/${rbname} $currentFile]
                set sec [expr [clock seconds] - (60 * 60)]
                if {[catch {exec sddsprocess -nowarn $currentFile -pipe=out -filter=column,Time,$sec,[clock seconds] | sdds2stream -pipe=in -col=Time} tsList]} {
                    APSSetVarAndUpdate status $tsList
                    return
                }
                set missed 0
                set t [lindex $tsList 0]
                set i 0
                set ts [lindex $tsList $i]
                while {$t < [lindex $tsList end]} {
                    if {$t != $ts} {
                        incr missed
                        set t [expr $t + $rate]
                        continue
                    }
                    incr i
                    set ts [lindex $tsList $i]
                    set t [expr $t + $rate]
                }
                set count [llength $tsList]
                if {$count == 0} {
                    set count 0
                    set missed 1
                }
                APSSetVarAndUpdate status "conditional${num}.${rate}${letter} [format %.2f [expr 100.0 * ($count) / ($count + $missed)]]% [format %6d $pvcount] PVs    $condition"
            }
        }
    }
}

proc CheckRunControls {args} {
    set list 0
    APSStrictParseArguments {list}
    if {$list} {
        APSSetVarAndUpdate status "Now listing Run Controls"
    } else {
        APSSetVarAndUpdate status "Now checking Run Controls"
    }
    if {[catch {sdds load /home/helios/oagData/dataLoggerConfig/timeSeries.config data} results]} {
        APSSetVarAndUpdate status "Error: $results"
    }
    set rootnameList [lindex $data(Column.rootname) 0]
    set rcPVList [lindex $data(Column.RunControlPV) 0]
    set doRunList [lindex $data(Column.doRun) 0]

    lappend rootnameList "Linac RF Faults Logger"
    lappend rcPVList OAG011RC 
    lappend doRunList 1
    lappend rootnameList "booster alarmlogger" "linac alarmlogger" "mps alarmlogger" "par alarmlogger" "sr alarmlogger" "timing alarmlogger" "srf alarmlogger" "acis alarmlogger" "runControl alarmlogger" "topup alarmlogger" "unconditional.16 logger" "SR converters glitchlogger" "lea  alarmlogger" "sr-ps alarmlogger" "S-rfns alarmlogger" "computeMotionInBands sdds" "computeMotionInBands band" "computeMotionInBands psd" "PSNoiseInBands sdds" "sddspcas PS noise" "PAR Timing feedforward" "SRIDLossToRfVoltageFF" "s-daq alarmlogger" "PSNoiseInBands band" "PSNoiseInBands psd"
    lappend rcPVList OAG099RC OAG098RC OAG097RC OAG096RC OAG095RC OAG094RC OAG093RC OAG092RC OAG090RC OAG089RC OAG120RC OAG105RC OAG087RC OAG088RC OAG091RC OAG150RC OAG148RC OAG149RC OAG151RC OAG152RC OAG190RC OAG191RC OAG086RC OAG153RC OAG154RC
    lappend doRunList 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 1 0 0

    #    lappend rootnameList "booster alarmlogger" "linac alarmlogger" "mps alarmlogger" "par alarmlogger" "sr alarmlogger" "timing alarmlogger" "srf alarmlogger" "acis alarmlogger" "srRTFB alarmlogger" "runControl alarmlogger" "topup alarmlogger" "SRDCPS-100Hz alarmlogger" "SCU6 alarmlogger" "unconditional.16 logger" "SCU1 alarmlogger" "SR converters glitchlogger"
    #    lappend rcPVList OAG099RC OAG098RC OAG097RC OAG096RC OAG095RC OAG094RC OAG093RC OAG092RC OAG091RC OAG090RC OAG089RC OAG088RC OAG119RC OAG120RC OAG122RC OAG105RC
    #    lappend doRunList 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1

    for {set i 0} {$i <= 22} {incr i} {
        #if {$i == 17} {continue}
        #if {$i == 20} {continue}
        #if {$i == 22} {continue}
        set num [format %03d $i]
        lappend rootnameList "gladiator CTL${num}RC datalogger"
        lappend rcPVList CTL${num}RC
        lappend doRunList 1
    }
    lappend rootnameList "turnByTurnTopup/scripts/collectInjectionMotion.runcontrol"
    lappend rcPVList OAG112RC 
    lappend doRunList 1
    lappend rootnameList turnByTurnTopup/scripts/processInjectionMotion.runcontrol
    lappend rcPVList OAG113RC 
    lappend doRunList 1
    lappend rootnameList PSSGlobalWatchdog
    lappend rcPVList OAG121RC 
    lappend doRunList 1
    lappend rootnameList OpsKickerLogging
    lappend rcPVList SR:UserOpsKickerLogRC
    lappend doRunList 0
    lappend rootnameList SCU6Feedforward
    lappend rcPVList OAG136RC
    lappend doRunList 0
    lappend rootnameList SCU6Feedforward
    lappend rcPVList OAG137RC 
    lappend doRunList 0
    lappend rootnameList MMSCorrection
    lappend rcPVList OAG138RC
    lappend doRunList 0
    lappend rootnameList "BBPM Waveform Collection"
    lappend rcPVList OAG139RC
    lappend doRunList 0
    lappend rootnameList "BEFI_MachineProtection"
    lappend rcPVList OAG017RC
    lappend doRunList 0
    lappend rootnameList "checkSumVerification"
    lappend rcPVList OAG019RC
    lappend doRunList 0
    lappend rootnameList "TBT-PostMortem FOK logger"
    lappend rcPVList OAG021RC
    lappend doRunList 0
    lappend rootnameList "maxRms sddspvalogger"
    lappend rcPVList OAG024RC
    lappend doRunList 1
    lappend rootnameList "aveSpectrum sddspvalogger"
    lappend rcPVList OAG025RC
    lappend doRunList 1
    lappend rootnameList "modeAmplitude sddspvalogger"
    lappend rcPVList OAG026RC
    lappend doRunList 1
    if {$list} {
        for {set i 0} {$i < 200} {incr i} {
            set num [format %03d $i]
            set j [lsearch -exact $rcPVList OAG${num}RC]
            if {$j != -1} {
                APSSetVarAndUpdate status "OAG${num}RC [lindex $rootnameList $j]"
            } else {
                if [catch {APScavget -noquote -list=OAG${num}RC.DESC} result] {
                }
                APSSetVarAndUpdate status "OAG${num}RC ------ $result"
            }
        }
        for {set i 0} {$i <= 30} {incr i} {
            set num [format %03d $i]
            set j [lsearch -exact $rcPVList CTL${num}RC]
            if {$j != -1} {
                APSSetVarAndUpdate status "CTL${num}RC [lindex $rootnameList $j]"
            } else {
                if [catch {APScavget -noquote -list=CTL${num}RC.DESC} result] {
                }
                APSSetVarAndUpdate status "CTL${num}RC ------ $result"
            }
        }
    } else {
        foreach rootname $rootnameList \
          rcPV $rcPVList \
          doRun $doRunList {
              if {$doRun == 0} {
                  continue
              }
              if {![llength $rcPV]} {
                  continue
              }
              if {$rootname == "SROrbit"} {
                  continue
              }
              if {$rootname == "srInjCycle"} {
                  continue
              }
              set rcPVorig ${rcPV}
              set rcPV ${rcPV}.RUN
              global $rcPV
              if {[pv linkw $rcPV $rcPV]} {
                  APSSetVarAndUpdate status "Error: Unable to connect to $rcPV used by the $rootname data logger"
              } else {
                  if {[pv getw $rcPV]} {
                      APSSetVarAndUpdate status "Error: Unable to connect to $rcPV used by the $rootname data logger"
                  } else {
                      if {[set $rcPV] != 1} {
                          APSSetVarAndUpdate status "$rootname data logger not running ($rcPV) $doRun"
                          catch {exec cavput -list=${rcPVorig}.ABRT=1,${rcPVorig}.CLR=1 -pend=.1}
                      }
                  }
              }
          }
    }
    APSSetVarAndUpdate status "Done"
}

proc CheckLoggerFiles {} {
    global expertMode verboseMode
    set tmpFile [APSTmpDir]/[APSTmpString]
    APSSetVarAndUpdate status "Checking... at [clock format [clock seconds]]"
    set oagDataHome "/home/helios/oagData/"
    if [catch {exec sddsprocess /home/helios/oagData/dataLoggerConfig/timeSeries.config \
                 -pipe=out -filter=column,doRun,1,1 \
                 | sddssort -pipe=in $tmpFile -column=workstation} result] {
        APSSetVarAndUpdate status $result
        return
    }
    if $verboseMode {
        APSSetVarAndUpdate status "Checking data-logger enable..."
    }
    if [catch {APScavget -noquote -list=DataLoggersOnOffBO} result] {
        APSSetVarAndUpdate status "$result"
        return
    }
    if [string compare $result "LOGGERS DISABLED"]==0 {
        LogOutputToLogDaemon -output "Loggers are disabled"
        APSSetVarAndUpdate status "Loggers are disabled!"
        APSSound -type emergency -volume 100
        return
    }
    if $verboseMode {
        APSSetVarAndUpdate status "Loggers are enabled."
    }
    if [catch {sdds open $tmpFile} inID] {
        APSSetVarAndUpdate status $inID
        catch {sdds close $inID}
        catch {exec rm $tmpFile}
        return
    }
    
    # If the storage ring operating mode is in "NO BEAM" or "MAINTENANCE", any datalogger  has
    # S35DCCT:CurrenCC or Mt:TopUpAutoEnableC should not be checked for updating. if Mt:TopUpAutoEnableC is
    # not 1 the top-up loggers are not expected to be updating.
    set SRoperating 1
    set TopUpEnabled 1
    catch {APScavget -noquote -list=S-DCCT:ActualMode } result
    APSSetVarAndUpdate status "S-DCCT:ActualMode = $result"
    if { [string compare "?" $result] == 0 } {
        set SRoperating 0
        APSSetVarAndUpdate status "Unable to get storage ring mode:$result"
    }
    if { [ string compare $result "NO BEAM" ]==0 ||\
           [ string compare $result "MAINTENANCE"]==0 ||\
           [ string compare $result "State Unknown"]==0 } {
        set SRoperating 0
    }	
    catch {APScavget -numerical -list=Mt:TopUpAutoEnableC } result
    APSSetVarAndUpdate status "Mt:TopUpAutoEnableC = $result"
    if { [string compare "?" $result] == 0 } {
        APSSetVarAndUpdate status "Unable to get top-up mode set bit:$result"
        set TopUpEnabled 0
    }
    if { [ string compare $result 0 ] == 0 } {
        set TopUpEnabled 0
    }	
    catch {exec cavget -provider=pva -list=S:InjCycleCaptureM.endTime -floatformat=%lf} result
    if { [string compare "?" $result] == 0 } {
        set SRInjTime [clock seconds]
    } else {
        set SRInjTime [expr int($result)]
    }
    

    
    foreach column {subDirectory rootname workstation loggerGroup RunControlPV extraArguments} {
        if [catch {sdds getColumn $inID $column} ${column}List] {
            catch {sdds close $inID}
            catch {exec rm $tmpFile}
            APSSetVarAndUpdate status [subst \$${column}List]
        }
    }
    lappend rootnameList SRFastLog1
    lappend workstationList charis
    lappend subDirectoryList monitoring/SRFastLog1
    lappend loggerGroupList ""
    lappend RunControlPVList ""
    lappend extraArgumentsList "-conditions=DCCT.mon,all"

    lappend rootnameList booster
    lappend workstationList nike
    lappend subDirectoryList Alarms/booster
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG099RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList linac
    lappend workstationList nike
    lappend subDirectoryList Alarms/linac
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG098RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList mps
    lappend workstationList nike
    lappend subDirectoryList Alarms/mps
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG097RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList par
    lappend workstationList nike
    lappend subDirectoryList Alarms/par
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG096RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList sr
    lappend workstationList nike
    lappend subDirectoryList Alarms/sr
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG095RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList timing
    lappend workstationList nike
    lappend subDirectoryList Alarms/timing
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG094RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList srf
    lappend workstationList nike
    lappend subDirectoryList Alarms/srf
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG093RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList acis
    lappend workstationList nike
    lappend subDirectoryList Alarms/acis
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG092RC"
    lappend extraArgumentsList ""
    
    #lappend rootnameList srRTFB
    #lappend workstationList nike
    #lappend subDirectoryList Alarms/srRTFB
    #lappend loggerGroupList ""
    #lappend RunControlPVList "OAG091RC"
    #lappend extraArgumentsList ""
    
    lappend rootnameList runControl
    lappend workstationList nike
    lappend subDirectoryList Alarms/runControl
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG090RC"
    lappend extraArgumentsList ""
    
    lappend rootnameList topup
    lappend workstationList nike
    lappend subDirectoryList Alarms/topup
    lappend loggerGroupList ""
    lappend RunControlPVList "OAG089RC"
    lappend extraArgumentsList ""
    
    #lappend rootnameList SRDCPS-100Hz
    #lappend workstationList nike
    #lappend subDirectoryList Alarms/SRDCPS-100Hz
    #lappend loggerGroupList ""
    #lappend RunControlPVList "OAG088RC"
    #lappend extraArgumentsList ""
    
    

    catch {sdds close $inID}
    catch {exec rm $tmpFile}
    set number [llength [subst $rootnameList]]
    
    set partialCheckGroups ""
    set partialCheckList ""
    if { !$SRoperating  || !$TopUpEnabled } {
        for { set index 0} {$index<$number} {incr index} {
            set extraArguments [lindex $extraArgumentsList $index]
            set subDirectory [lindex $subDirectoryList $index]
            set loggerGroup [lindex $loggerGroupList $index]
            set partialCheck 0 
            set  tmpindex0 [ string first "-cond" $extraArguments]
            if { $tmpindex0 != -1} {
                
                set tmpindex1 [ string first "=" $extraArguments $tmpindex0]
                set tmpindex2 [ string first "," $extraArguments $tmpindex1]
                set conditionFile [ string range $extraArguments [expr $tmpindex1+1] [expr  $tmpindex2-1 ] ]
                
                if  { [ string first "/" $conditionFile ] == -1 } {
                    set conditionFile ${oagDataHome}$subDirectory/$conditionFile 
                }    
                if {[catch {exec sdds2stream $conditionFile -col=ControlName} controlNameList]} {
                    APSSetVarAndUpdate status $controlNameList
                }
                if { !$SRoperating } {
                    if { ([ lsearch -glob $controlNameList "S-DCCT:CurrentM*"] != -1) || \
                           ([ lsearch -glob $controlNameList "Mt:TopUpAutoEnableC*"] != -1) } {
                        set partialCheck 1
                        if { [ llength $loggerGroup ] } {   
                            lappend partialCheckGroups $loggerGroup
                        }
                    }
                } else { 
                    if { [ lsearch -glob $controlNameList "Mt:TopUpAutoEnableC*"] != -1 } {
                        set partialCheck 1
                        if { [ llength $loggerGroup ] } {
                            lappend partialCheckGroups $loggerGroup
                        }
                    }   
                }
            }
            if {$subDirectory == "monitoring/SRDCPS-HVD-100Hz"} {
                set partialCheck 1
            }
            if {$subDirectory == "monitoring/SRDCPS-QS-100Hz"} {
                set partialCheck 1
            }
            if {$subDirectory == "logging/srInjCycle"} {
                set partialCheck 1
            }
            if {$subDirectory == "logging/BoosterWF"} {
                set partialCheck 1
            }
            lappend partialCheckList $partialCheck  
        }
        
    }
    
    set tag [exec date +%Y-%j]
    set errors 0
    set cronProbs 0
    set timeNow [expr int([exec timeconvert -now | token -last])]
    set cronProbList ""
    set oldFiles 0
    set notRunning 0
    set oldFilesInj 0
    set oldFilesTopUp 0
    set oldFilesLogOnChange 0
    set noData 0
    for {set index 0} {$index<$number} {incr index} {
        update
        set problem 0
        set rootname [lindex $rootnameList $index]
        set subDirectory [lindex $subDirectoryList $index]
        set pattern /home/helios/oagData/$subDirectory/$rootname-$tag-????.*
        set workstation [lindex $workstationList $index]
        set loggerGroup [lindex $loggerGroupList $index]
        set extraArguments [lindex $extraArgumentsList $index]
        set partialCheck   [lindex $partialCheckList $index]
        set rcPV [lindex $RunControlPVList $index]
        if {$subDirectory == "logging/srBPMAve"} {continue}
        if $verboseMode {
            APSSetVarAndUpdate status "Checking $rootname..."
        }
        set dataFiles [lsort [glob -nocomplain $pattern]]
        set count [llength $dataFiles]
        if !$count {
            if {!$SRoperating && $expertMode && (([string compare $subDirectory "logging/linacRFWF"]==0) || 
                                                 ([string compare $subDirectory "logging/parRFWF"]==0) ||
                                                 ([string compare $subDirectory "logging/BunchPurityWF"]==0) ||
                                                 ([string compare $subDirectory "logging/srInjCycle"]==0))} {
                #ignore
            } elseif {!$TopUpEnabled && ([string compare $subDirectory "logging/srInjCycle"]==0)} {
                #ignore
            } elseif {([string compare $subDirectory "glitchLogs/SRvac"]==0)} {
                #ignore
            } else {
                LogOutputToLogDaemon -output "${subDirectory}/${rootname}@${workstation}: no data"
                APSSetVarAndUpdate status "${subDirectory}/${rootname}@${workstation}: no data !"
                incr errors 
                incr noData
            }
            continue
        }
        if $verboseMode {
            APSSetVarAndUpdate status "Found $count files for $tag"
        }
        if {$expertMode && \
              [string compare [file dirname $subDirectory] "glitchLogs"]!=0 && \
              [string compare [file dirname $subDirectory] "logonchange"]!=0 && \
              [string compare [file dirname $subDirectory] "Alarms"]!=0 && \
              [string compare $subDirectory "logging/linacRFWF"]!=0 && \
              [string compare $subDirectory "logging/parRFWF"]!=0 && \
              [string compare $rootname "SRBunchPurity"]!=0 && \
              [string compare $rootname "BunchPurityWF"]!=0} {
            # don't check data files for glitch logs or log-on-change files as they won't update
            # if there are no glitches or changes
            set partialCheck 0 
            if { !$SRoperating  || !$TopUpEnabled } {
                set partialCheck   [lindex $partialCheckList $index]
                if { [lsearch $partialCheckGroups $loggerGroup ] != -1  } {
                    set partialCheck 1
                }
                
            }
            if { $partialCheck == 0 } {	       	    
                set mtime [file mtime [lindex $dataFiles end]]
                if {${rootname} == "srInjCycle"} {
                    if {[expr $mtime+15*60] < $SRInjTime } {
                        LogOutputToLogDaemon -output "${subDirectory}/${rootname}@${workstation} not updated recently"
                        APSSetVarAndUpdate status "${subDirectory}/${rootname}@${workstation} not updated recently"
                        set problem 1
                        incr oldFiles
                        lappend oldFileWS $workstation
                    }

                } else {
                    if {[expr $mtime+15*60]<$timeNow } {
                        LogOutputToLogDaemon -output "${subDirectory}/${rootname}@${workstation} not updated in last 15 minutes"
                        APSSetVarAndUpdate status "${subDirectory}/${rootname}@${workstation} not updated in last 15 minutes"
                        if [string match *Injection* [file tail [lindex $dataFiles end]]] {
                            incr oldFilesInj 
                        }
                        if [string match *TopUp* [file tail [lindex $dataFiles end]]] {
                            incr oldFilesTopUp
                        }
                        set problem 1
                        incr oldFiles
                        lappend oldFileWS $workstation
                    }
                }
            }
        }
        if {[string range $subDirectory 0 5] == "Alarms"} {
            set logFile /home/helios/oagData/Alarms/$rootname.logfile
        } else {
            set logFile /home/helios/oagData/$subDirectory/$rootname.logfile
            set logFile2 /home/helios/oagData/logging/$workstation.logfile${loggerGroup}
            if [file exists $logFile2] {
                set logFile $logFile2
            }
        }
        if ![file exists $logFile] {
            LogOutputToLogDaemon -output "${subDirectory}/${rootname}@${workstation}: no logfile found"
            APSSetVarAndUpdate status "${subDirectory}/${rootname}@${workstation}: no logfile found"
            incr cronProbs
        } else {
            if $verboseMode {
                APSSetVarAndUpdate status "Log file was found"
            }
            if {![llength $rcPV]} {
                set mtime [file mtime $logFile]
                if {[expr $mtime+15*60]<$timeNow} {
                    LogOutputToLogDaemon -output "${subDirectory}/${rootname}@${workstation}: logfile ($logFile) not updating."
                    APSSetVarAndUpdate status "${subDirectory}/${rootname}@${workstation}: logfile ($logFile) not updating."
                    incr cronProbs
                    if {![llength $cronProbList] || \
                          [lsearch $cronProbList $workstation]==-1} {
                        lappend cronProbList $workstation
                    }
                }
            } else {
                set rcPV ${rcPV}.RUN
                global $rcPV
                if {[pv linkw $rcPV $rcPV]} {
                    APSSetVarAndUpdate status "Error: Unable to connect to $rcPV used by the $rootname data logger"
                } else {
                    if {[pv getw $rcPV]} {
                        APSSetVarAndUpdate status "Error: Unable to connect to $rcPV used by the $rootname data logger"
                    } else {
                        if {[set $rcPV] != 1} {
                            if {[string compare $rootname "srInjCycle"]!=0} {
                                APSSetVarAndUpdate status "${subDirectory}/$rootname data logger not running"
                                incr notRunning
                                if (!$problem) {
                                    incr oldFiles
                                }
                            }
                        }
                    }
                }
            }
        }
    }
    if {($oldFiles || $notRunning) && $expertMode} {
        APSSetVarAndUpdate status "$notRunning data loggers is not running properly."
        APSSetVarAndUpdate status "$oldFiles files *may* not be updating properly."
        if $oldFilesInj {
            APSSetVarAndUpdate status "* $oldFilesInj of these are injection logs."
            APSSetVarAndUpdate status "  This is normal if there was no injection in the last 15 minutes"
            APSSetVarAndUpdate status "  or during top-up."
        }
        if $oldFilesTopUp {
            APSSetVarAndUpdate status "* $oldFilesTopUp of these are top-up logs."
            APSSetVarAndUpdate status "  This is normal if you are not in top-up mode."
        }
    }
    if {[expr $oldFiles-$oldFilesInj-$oldFilesTopUp]==0 && !$errors && !$cronProbs && !$notRunning} {
        APSSetVarAndUpdate status "No definite problems detected with $number loggers."
    } else {
        APSSetVarAndUpdate status "$errors loggers have no data."
        APSSetVarAndUpdate status "$cronProbs loggers have cron job problems."
        APSSetVarAndUpdate status "[expr $oldFiles-$oldFilesInj-$oldFilesTopUp] normal loggers have problems."
        if $cronProbs {
            set cronProbList [lsort $cronProbList]
            APSSetVarAndUpdate status "* Suggest contacting Bob Soliday to fix cron processes"
            APSSetVarAndUpdate status "  on the following workstations:"
            APSSetVarAndUpdate status "  [join $cronProbList ,]"
        }
        if [expr $oldFiles-$oldFilesInj-$oldFilesTopUp]>=1 {
            APSSetVarAndUpdate status "* [expr $oldFiles-$oldFilesInj-$oldFilesTopUp] normal logs are not"
            APSSetVarAndUpdate status "  updating.  If there is stored beam, try this check again"
            APSSetVarAndUpdate status "  in a few minutes.  If problems persist, contact OAG."
        }
        if $noData {
            APSSetVarAndUpdate status "****\n$noData loggers have no data.  Contact OAG immediately.\n****"
        }
        APSSound -type emergency -volume 100
    }
}

proc LogOutputToLogDaemon {args} {
    global apsScriptUser apsScriptHost
    APSStrictParseArguments {output}
    if {[catch {exec logMessage -sourceId=applicationErrors \
                  -tag=User "$apsScriptUser" \
                  -tag=Host "$apsScriptHost" \
                  -tag=Application "checkDataLoggers" \
                  -tag=Error "$output"} result]} {
        APSSetVarAndUpdate status "error: $result"
    }
}

proc ViewLog {args} {
    set logDir /home/helios/oagData/logDaemonData
    set log applicationErrors
    set sec [clock seconds]
    set sec2 [expr $sec - {60 * 60 * 24 * 7}]
    set fileList  \
      [APSFindFilesBetweenDates \
         -tailsOnly 0 \
         -extensionList {"*"}\
         -directory $logDir/$log \
         -startDateList [APSFormatDate \
                           -year [clock format $sec2 -format %Y] \
                           -month [string trimleft [clock format $sec2 -format %m] 0] \
                           -day [clock format $sec2 -format %e] \
                           -dateFormat list] \
         -endDateList [APSFormatDate \
                         -year [clock format $sec -format %Y] \
                         -month [string trimleft [clock format $sec -format %m] 0] \
                         -day [clock format $sec -format %e] \
                         -dateFormat list] ]
    if {![llength $fileList]} {
        APSSetVarAndUpdate status "Log files for $log not found."
        return
    }

    if {[catch {eval exec sddscombine $fileList -pipe=out -merge | sddsprocess -pipe -match=column,Application=checkDataLoggers -nowarn | sddstimeconvert -pipe -breakdown=column,Time,text=TimeString | sddsprintout -pipe=in "-columns=(User,Host,TimeString,Error),format=%s" -noLabels -notitle} results]} {
        APSSetVarAndUpdate status "error: $results"
    }
    APSSetVarAndUpdate status $results
}
