
# Removes data files that are older than a specified age.
proc APSRemoveExpiredTimeSeriesData {args} {
    APSDefineMonitoringVariables
    global apsOAGDataDir
    global apsDataManagerNormalEmail apsDataManagerPagerEmail
    global apsDataManagerNormalEmailOn apsDataManagerPagerEmailOn

    set subDirectory ""
    set rootname ""
    if [APSStrictParseArguments {subDirectory rootname}]==-1 {
        return -code error "APSRemoveExpiredTimeSeriesData: unrecognized arguments"
    }
    if {![string length $subDirectory] || ![string length $rootname]} {
        return -code error "APSRemoveExpiredTimeSeriesData: invalid arguments"
    }
    
    # this call checks for the instruction file, gets the contents for this
    # logger, and sets variables in this routine based on those contents.
    # variables defined are: doRun sampleInterval extraArguments workstation processingScript monitorProgram keepGenerations
    if [catch {APSReadTimeSeriesLoggerConfig \
                 -subDirectory $subDirectory -rootname $rootname} result] {
        return -code error "APSRemoveExpiredTimeSeriesData: $result"
    }
    if $daysToSave<0 {
        return
    }

    set oldDir [pwd]
    cd $apsOAGDataDir/$subDirectory

    set fileList [lsort -increasing \
                    [glob -nocomplain $rootname-????-???-????.gz]]
    if [llength $fileList]<=$daysToSave {
        cd $oldDir
        return
    }
    set expiredList [lrange $fileList 0 [expr int([llength $fileList]-$daysToSave-1)]]

    foreach fileName $expiredList {
        if [string compare [file type $fileName] link] {
            lappend deleteList $fileName
        } else {
            lappend deleteList $fileName [APSResolveLink $fileName]
        }
    }
    if [catch {eval file delete -force $deleteList} result] {
        return -code error "APSRemoveExpiredTimeSeriesData: $result"
    }

    set logfile $rootname.plogfile
    APSAddToMonitoringLogFile -fileName $logfile -lineLimit 500 -text "Deleted expired files: $deleteList"
}

# Combines generation files into single result, saves
# generation files, and runs optional processing script.
proc APSPostprocessScalarMonitorData {args} {
    APSDefineMonitoringVariables
    global apsOAGDataDir apsScriptHost
    global apsDataManagerNormalEmail apsDataManagerPagerEmail
    global apsDataManagerNormalEmailOn apsDataManagerPagerEmailOn

    set subDirectory ""
    set rootname ""
    set processingScript ""
    set dateString ""
    set niceness 19
    if [APSStrictParseArguments {subDirectory rootname processingScript dateString niceness}]==-1 {
        return -code error "APSPostprocessScalarMonitorData: unrecognized arguments"
    }
    if {![string length $subDirectory] || ![string length $rootname]} {
        return -code error "APSPostprocessScalarMonitorData: invalid arguments"
    }

    # this call checks for the instruction file, gets the contents for this
    # logger, and sets variables in this routine based on those contents.
    # variables defined are: doRun sampleInterval extraArguments workstation processingScript monitorProgram keepGenerations globalProcessingScript postprocessingWorkstation
    if [catch {APSReadTimeSeriesLoggerConfig \
                 -subDirectory $subDirectory -rootname $rootname} result] {
        return -code error "APSPostprocessScalarMonitorData: $result"
    }
    if [string compare [lindex [split $apsScriptHost .] 0] $postprocessingWorkstation] {
        return -code error "Warning: attempt to process data on workstation [lindex [split $apsScriptHost .] 0] that is intended for workstation $postprocessingWorkstation\n$subDirectory $rootname"
    }
    set oldDir [pwd]
    cd $apsOAGDataDir/$subDirectory

    # check/establish processing lock
    if [catch {APSCreateLockFile -fileName $rootname.plock} lockFID] {
        return -code error "APSPostprocessScalarMonitorData: $lockFID"
    }

    # prepare the procesing log file
    set logfile $rootname.plogfile
    APSAddToMonitoringLogFile -fileName $logfile -lineLimit 500 -text "Processing files"

    #
    # process (or attempt to process) files from last 30 days, 
    # starting with the oldest, ending yesterday
    #
    set noFilesTagList ""
    APSDateBreakDown -yearVariable year -twoDigitYear 0 -julianDayVariable jday 
    set dayOffset -7
    set myID "[lindex [split $apsScriptHost .] 0] [pid]"
    while 1 {
        incr dayOffset
        if {$dayOffset==0} break

        set tag [APSOffsetDateInfo -offset $dayOffset -today 1 -dateFormat Y-J-MD]
        
        if [string length $dateString] {
            # process the single date given by the dateString string, which contains a string of
            # the form YYYY-JJJ
            set flist [glob -nocomplain $rootname-$dateString-\[0-9\]\[0-9\]\[0-9\]\[0-9\].\[0-9\]\[0-9\]\[0-9\]\[0-9\]]
            set count [llength $flist]
            if $count {
                set flist [lsort $flist]
                # construct the full date tag and use to make the output rootname
                set year [lindex [split $dateString -] 0]
                set jday [lindex [split $dateString -] 1]
                set tag [APSOffsetDateInfo -offset 0 -julianDay $jday -year $year -dateFormat Y-J-MD]
                set outputRoot $rootname-$tag
            }
            # this will result in termination of the loop after processing 
            # the current batch of data
            set dayOffset -1
        } else {
            # make list of output files for this date
            set outputRoot "$rootname-${tag}"
            set flist [glob -nocomplain ${outputRoot}.\[0-9\]\[0-9\]\[0-9\]\[0-9\]]
            set count [llength $flist]
            if {!$count} {
                if ![file exists $outputRoot.gz] {
                    lappend noFilesTagList $tag
                }
            } else {
                set flist [lsort $flist]
            }
        }

        if !$count {
            if [string length $dateString] {
                cd $oldDir
                close $lockFID
                return -code error "$myID: APSPostprocessScalarMonitorData: No files: $rootname $dateString"
            }
            continue
        }

        # make a list of good and bad files
        set goodlist {}
        set badlist {}
        foreach file $flist {
            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Checking file $file"
            catch {exec sddscheck $file} check
            switch $check {
                "nonexistent" -
                "badHeader" {
                    lappend badlist $file
                }
                "corrupted" {
                    # recover corrupted file
                    APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Recovering corrupted file: $file" 
                    if [catch {exec sddsconvert -recover $file -nowarning} result] {
                        catch {sddscheck $file} result1
                        if {[catch {exec sddscheck $file} result1] || [string compare $result1 ok]!=0} {
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Recovery result: $result--$result1"
                            lappend badlist $file
                        } else {
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Recovery result: $result--$result1"
                            lappend goodlist $file
                            catch {file delete ${file}~}
                        }
                    } else {
                        APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Recovery result: $result"
                        lappend goodlist $file
                    }
                }
                default {
                    lappend goodlist $file
                }
            }
        }

        # record good/bad file information
        if [llength $badlist] {
            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Bad files: $badlist"
            APSSendEMail -address $apsDataManagerNormalEmail -message "$myID: APSPostprocessScalarMonitorData: Bad files $tag $rootname: $badlist"
        }
        if [llength $goodlist] { 
            set failed 0
            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Processing [llength $goodlist] files."


            if {[catch {exec sddsquery [lindex $goodlist 0] -parameterlist} parameterlist]} {
                set failed 1
                APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $parameterlist"
            }
            if {$failed == 0} {
                if {[catch {exec sddsquery [lindex $goodlist 0] -columnlist} columnlist]} {
                    set failed 1
                    APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $columnlist"
                }
            }
            if {$failed == 0} {
                if [llength $goodlist]>1 {
                    if {[lsearch -exact $columnlist "Time"] >= 0} {
                        if [catch {eval exec sddscombine $goodlist -pipe=out \
                                     | sddsprocess -pipe=in ${outputRoot}.gz \
                                     -filter=column,Time,1e8,1e100 \
                                     -nowarning \
                                     {{-test=param,n_rows 0 >}} } result] {
                            set failed 1
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $result"
                        }
                    } elseif {[lsearch -exact $parameterlist "Time"] >= 0} {
                        if [catch {eval exec sddscombine $goodlist ${outputRoot}.gz} result] {
                            set failed 1
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $result"
                        }
                    } else {
                        if [catch {eval exec sddscombine $goodlist -pipe=out \
                                     | sddsprocess -pipe=in ${outputRoot}.gz \
                                     -nowarning \
                                     {{-test=param,n_rows 0 >}} } result] {
                            set failed 1
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $result"
                        }
                    }
                } else {
                    if {[lsearch -exact $columnlist "Time"] >= 0} {
                        if [catch {eval exec sddsprocess $goodlist ${outputRoot}.gz \
                                     -filter=column,Time,1e8,1e100 \
                                     -nowarning \
                                     {{-test=param,n_rows 0 >}} } result] {
                            set failed 1
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $result"
                        }
                    } elseif {[lsearch -exact $parameterlist "Time"] >= 0} {
                        if [catch {eval exec sddsprocess $goodlist ${outputRoot}.gz \
                                     -nowarning } result] {
                            set failed 1
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $result"
                        }
                    } else {
                        if [catch {eval exec sddsprocess $goodlist ${outputRoot}.gz \
                                     -nowarning \
                                     {{-test=param,n_rows 0 >}} } result] {
                            set failed 1
                            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $result"
                        }
                    }
                }
            }

            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: failed = $failed"
            if {!$failed && [file exists $outputRoot.gz] && [file size $outputRoot.gz]} {
                APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Good files: $goodlist --\> ${outputRoot}.gz"
            } else {
                set message "Processing failed.  Original files left intact.  $result" 
                APSAddToMonitoringLogFile -fileName $logfile -text "$myID: $message"
                catch {file delete -force ${outputRoot}.gz}
                APSSendEMail -address $apsDataManagerNormalEmail \
                  -message "$myID: APSPostprocessScalarMonitorData: Processing failed.  Original files left intact.  $tag $rootname"
                if [string length $dateString] {
                    cd $oldDir
                    close $lockFID
                    return -code error $message
                }
                continue
            }
        }

        if $keepGenerations {
            # compress all files and move to generations subdirectory
            if {([file exists generations] && ![file isdirectory generations] && \
                   ([catch {file delete -force generations} result] || [catch {exec mkdir generations} result])) || \
                  (![file exists generations] && [catch {exec mkdir generations} result])} {
                set message "$myID: APSPostprocessScalarMonitorData: Problem with generations file: $result"
                APSAddToMonitoringLogFile -fileName $logfile -text $message
                APSSendEMail -address $apsDataManagerNormalEmail -message $message
            }
            if [file exists generations] {
                APSAddToMonitoringLogFile -fileName $logfile -text "$myID: stashing [llength $flist] generations files"
                foreach file $flist {
                    set message ""
                    if [catch {exec gzip $file} result] {
                        set message "$myID: gzip on $file: $result"
                    } else {
                        if {([file exists $file.gz] && [catch {exec mv $file.gz generations/} result]) || \
                              ([file exists $file] && [catch {exec mv $file generations/} result])} {
                            set message "$myID: APSPostprocessScalarMonitorData: mv error for $subDirectory/$rootname: $result"
                        }
                    }
                    if [string length $message] {
                        APSAddToMonitoringLogFile -fileName $logfile -text $message
                        APSSendEMail -address $apsDataManagerNormalEmail -message $message
                    }
                }
            }
        } else {
            # remove original files, but only if output file is ok
            if {[file exists ${outputRoot}.gz] && [APSCheckSDDSFile -fileName ${outputRoot}.gz]} {
                APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Removing original files" 
                if [catch {eval file delete -force $flist} result] {
                    APSSendEMail -address $apsDataManagerNormalEmail -message \
                      "$myID: APSPostprocessScalarMonitorData: rm error for $subDirectory/$rootname: $result"
                }
            }
        }

        APSAddToMonitoringLogFile -fileName $logfile -text "$myID: Done collating [exec date]"

        if [file exists ${outputRoot}.gz] {
            # run custom script, if given
            if [string length $processingScript] {
                if [catch {exec $processingScript -rootname $rootname -subDirectory $subDirectory \
                             -dateTag $tag -fileName $outputRoot.gz } result] {
                    set message "$myID: APSPostprocessScalarMonitorData: Script $processingScript returned error: $result" 
                    APSAddToMonitoringLogFile -fileName $logfile -text $message
                    APSSendEMail -address $apsDataManagerNormalEmail -message $message
                    continue
                } else {
                    APSAddToMonitoringLogFile -fileName $logfile -text \
                      "$myID: Done processing with $processingScript [exec date]"
                }
            }

            if {[file exists columnIndices] && [file isdirectory columnIndices]} {
                if {[catch {exec chmod a-w ${outputRoot}.gz} result] || \
                      [catch {exec sddsquery ${outputRoot}.gz -column -sddsOutput \
                                | sddssort -pipe -column=Name \
                                | gzip -c > columnIndices/${outputRoot}.index.gz} result]} {
                    APSAddToMonitoringLogFile -fileName $logfile -text \
                      "$myID: APSPostprocessScalarMonitorData: Problem making column index file: $result"
                }
            }
        }
    }

    if [string length $globalProcessingScript] {
        if [catch {eval exec $globalProcessingScript \
                     -subDirectory $subDirectory \
                     -rootname $rootname } result] {
            set message "Problem with global processing: $result"
            APSAddToMonitoringLogFile -fileName $logfile -text "$myID: APSPostprocessScalarMonitorData: $message"
            APSSendEMail -address $apsDataManagerNormalEmail -message \
              "$myID: $subDirectory/$rootname: $message"
        }
    }

    if [llength $noFilesTagList] {
        APSAddToMonitoringLogFile -fileName $logfile -text "$myID: APSPostprocessScalarMonitorData: No files: $subDirectory/$rootname: $noFilesTagList"
    }
    close $lockFID
}

proc APSPostprocessChargeFastLog {args} {
    set dataDir /home/helios/oagData/monitoring/Charge

    set positrons 1
    set targetDir $dataDir
    set force 0
    set debug 0
    set year ""
    set julianDay ""
    set tag ""
    set first 0
    APSParseArguments {positrons targetDir force debug year julianDay tag first}

    if [string length $tag]!=13 {
        if {![string length $year] || ![string length $julianDay] || \
              $year<=0 || $julianDay<=0 || $julianDay>366} {
            return -code error "APSPostprocessChargeFastLog: bad date info: year=$year julianDay=$julianDay tag=$tag"
        }
        set dateList [APSFormatDate -julianDay $julianDay -year $year -dateFormat list]
        set year [lindex $dateList 0]
        set julianDay [lindex $dateList 1]
        set MM [lindex $dateList 2]
        set DD [lindex $dateList 3]
        set tag $year-$julianDay-${MM}${DD}
        if $debug {
            puts stderr "Year/jd=$year/$julianDay = $tag"
        }
    } else {
        if $debug { puts stderr "tag: $tag" }
        scan $tag %ld-%ld-%02ld%02ld year julianDay MM DD
        set MM [format %02ld $MM]
        set DD [format %02ld $DD]
        if $debug { puts stderr "year/yd/mm/dd = $year/$julianDay/$MM/$DD" }
    }

    set pfile $targetDir/ChargeFastProc-${tag}.gz
    if $debug {
        puts stderr "working on $pfile"
    }

    if {[string compare $dataDir $targetDir]==0 && [file exists $pfile] && !$force} { 
        if $debug {
            puts stderr "$pfile exists"
        }
        return
    }
    catch {exec whoami} user
    if {[string compare $targetDir $dataDir]==0 && [string compare $user "oag"]!=0} {
        if $debug {
            puts stderr "no access to directory"
        }
        return -code error "APSPostprocessChargeFastLog: No access to directory $dataDir"
    }

    set maxReadout 300
    set timeStart [expr 5.0/60.0/24.0]
    set timeEnd   [expr $timeStart+24.0]

    if !$positrons {
        set sign -1
        set maxLimit 0
        set minLimit -$maxReadout
    } else {
        set sign 1
        set maxLimit $maxReadout
        set minLimit 0
    }

    set rootname $dataDir/ChargeFast-${tag}
    set source /tmp/[APSTmpString]
    set tmpID [APSUniqueName tmp]
    APSAddToTmpFileList -ID $tmpID -fileList $source
    set rawFiles [glob -nocomplain $rootname.\[0-9\]\[0-9\]\[0-9\]\[0-9\] ]
    if [llength $rawFiles] {
        eval exec sddscombine $rawFiles -pipe=out \
          | sddsconvert -pipe=in $source \
          "-retain=col,Time*,*DCCT*,DayOfMonth,ShutterPermit"  \
          "-rename=column,S35:DCCT=S35DCCT,S:DCCT=S35DCCT" 
    } elseif [file exists ${rootname}] {
        exec sddsconvert -recover ${rootname} $source  \
          -retain=col,Time*,*DCCT*,DayOfMonth,ShutterPermit  \
          "-rename=column,S35:DCCT=S35DCCT,S:DCCT=S35DCCT" 
    } elseif [file exists ${rootname}.gz] {
        exec gzip -dcn ${rootname}.gz \
          | sddsconvert -recover -pipe=in $source \
          -retain=col,Time*,*DCCT*,DayOfMonth,ShutterPermit \
          "-rename=column,S35:DCCT=S35DCCT,S:DCCT=S35DCCT" 
    } else {
        if $debug {
            puts stderr "Source file $rootname not found"
        }
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSPostprocessChargeFastLog: Source file $rootname not found"
    }
    if $debug { puts stderr "Made source file" }

    set tmpFile /tmp/[APSTmpString]
    APSAddToTmpFileList -ID $tmpID -fileList \
      [list $tmpFile.data $tmpFile.integ $tmpFile.integ1 $tmpFile.param $tmpFile.data1 \
         $tmpFile.ave $tmpFile.life]
    set dateList [APSPreviousDaysDate -year $year -julianDay $julianDay -twoDigitYear 0 \
                    -dateFormat list]
    set previousTag [lindex $dateList 0]-[lindex $dateList 1]-[lindex $dateList 2][lindex $dateList 3]
    set previousFile $dataDir/ChargeFastProc-${previousTag}.gz
    if ![file exists $previousFile] {
        puts stderr "Warning: no previous file ($previousFile) for date $MM/$DD/$year (JD=$julianDay)"
        set previousIntegral 0
        if !$first {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSPostprocessChargeFastLog: no previous file ($previousFile) for date $MM/$DD/$year (JD=$julianDay)"
        }
    } else {
        set data [APSGetSDDSParameter -parameter S35DCCTGrandIntegDay -fileName $previousFile]
        set previousIntegral [lindex $data end]
    }
    if $debug { puts stderr "previous integral is $previousIntegral" }

    # Despike the data and break into segments, then filter out out-of-range current values.
    # Tee data off after this.
    # Next, integrate the segments and combine to get integral for the day.
    # Finally, add some parameters from the source file to create a file of parameters only.
    if [catch \
          {exec sddsbreak $source -pipe=out -gapin=TimeOfDay,factor=1.5 \
             | sddssmooth -despike=neighbor=10,pass=1 -pass=0 -pipe -column=S35DCCT \
             | sddsprocess -pipe -nowarning \
             "-redefine=col,S35DCCT,S35DCCT $maxLimit > S35DCCT $minLimit < || ? 0 : S35DCCT $ abs " \
             "-redefine=col,TimeOfDay,TimeOfDay,units=h,description=,symbol=t" \
             "-filter=column,TimeOfDay,$timeStart,$timeEnd" -nowarn \
             "-test=param,n_rows 0 >" \
             | tee ${tmpFile}.data \
             | sddsinteg -pipe -versus=TimeOfDay -integrate=S35DCCT \
             | sddsprocess -pipe "-rpnexpr=0 sto PreviousSegment" -nowarning \
             "-redefine=column,S35DCCTInteg,S35DCCTInteg PreviousSegment +,units=mA-h" \
             "-define=column,S35DCCTGrandInteg,S35DCCTInteg $previousIntegral +,units=mA-h" \
             "-process=S35DCCTInteg,last,S35DCCTIntegSegment" \
             "-process=S35DCCTGrandInteg,last,S35DCCTGrandIntegSegment" \
             "-rpnexpr=S35DCCTIntegSegment sto PreviousSegment,repeat" \
             | tee ${tmpFile}.integ1  \
             | sddscollapse -pipe \
             | sddsprocess -pipe -nowarning -process=S35DCCTIntegSegment,last,S35DCCTIntegDay \
             "-process=S35DCCTGrandIntegSegment,last,S35DCCTGrandIntegDay" \
             | sddsxref -pipe $source -leave=* -transfer=param,TimeStamp,StartTime,YearStartTime \
             -nowarning -reuse=page \
             | sddsprocess -pipe=in $tmpFile.param -nowarning \
             "-define=parameter,DayOfYear,StartTime YearStartTime - 8.64e4 /,type=long" \
             -define=parameter,DayOfMonth,$DD,type=long \
             -define=parameter,Month,$MM,type=long \
             "-print=parameter,DayLabel,%ld/%ld,Month,DayOfMonth"} result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSPostprocessChargeFastLog: pipeline 1 error: $result"
    }
    if $debug { puts stderr "First pipeline done" }

    if [catch {exec sddscombine ${tmpFile}.integ1 ${tmpFile}.integ -merge} result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSPostprocessChargeFastLog: $result"
    }
    if $debug { puts stderr "Integral file merged" }

    # See if column ShutterPermit exists.  If not, define it with value 0.
    set oldDir [pwd]
    cd /tmp
    catch {exec sddsprocess ${tmpFile}.data \
             -ifnot=column,ShutterPermit \
             -define=column,ShutterPermit,0 -nowarning } 
    cd $oldDir
    if $debug { puts stderr "ShutterPermit test/definition done."}

    # Merge the segmented data and compute the average current over all samples.
    # At the same time, figure out the percent of the time the beam is >1mA
    if [catch \
          {exec sddscombine ${tmpFile}.data -pipe=out -merge \
             | tee ${tmpFile}.data1 \
             | sddsprocess -pipe=in ${tmpFile}.ave -nowarn \
             -process=S35DCCT,largest,S35DCCTPeak -process=S35DCCT,ave,S35DCCTAverage \
             "-define=col,AbsS35DCCT,S35DCCT abs" \
             {-define=column,BeamOver1mA,AbsS35DCCT 1 > ? 1 : 0 $,type=long} \
             "-define=column,BeamDelivered,BeamOver1mA ShutterPermit *,type=long"  \
             "-process=BeamOver1mA,sum,SamplesOver1mA" \
             "-process=BeamDelivered,sum,SamplesDelivered" \
             "-define=parameter,BeamOver1mAPercent,SamplesOver1mA n_rows / 100 *,units=%" \
             "-define=parameter,BeamDeliveredPercent,SamplesDelivered n_rows / 100 *,units=%" } result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSPostprocessChargeFastLog: $result"
    }
    
    if $debug { puts stderr "Current stats done " }

    # Compute the lifetime for each segment separately
    if [catch \
          {exec sddsderiv ${tmpFile}.data -pipe=out -versus=TimeOfDay \
             -differ=S35DCCT -interval=2 \
             | sddsxref -pipe ${tmpFile}.data -take=S35DCCT,ShutterPermit -nowarn \
             | sddsprocess -pipe -nowarn \
             "-define=column,AbsS35DCCT,S35DCCT abs" \
             "-define=column,Lifetime,S35DCCT S35DCCTDeriv * 0 < ? S35DCCT S35DCCTDeriv / chs : 0 $ ,units=h" \
             | sddssmooth -despike=neighbor=5,pass=2 -pass=0 -column=Lifetime -pipe \
             | sddscombine -merge -pipe \
             | sddsprocess -pipe  -nowarn \
             -process=Lifetime,ave,LifetimeAverage,functionOf=AbsS35DCCT,lower=1 \
             "-redefine=param,LifetimeAverage,LifetimeAverage 200 > ? 0 : LifetimeAverage $ ,symbol=Average Lifetime for >1mA,units=h" \
             -process=Lifetime,max,LifetimePeak,functionOf=AbsS35DCCT,lower=1  \
             "-redefine=param,LifetimePeak,LifetimePeak 200 > ? 0 : LifetimePeak $ ,symbol=Peak Lifetime >1mA,units=h" -nowarning \
             | sddsconvert -pipe=in ${tmpFile}.life -retain=column,S35DCCT,Lifetime,TimeOfDay,ShutterPermit} result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSPostprocessChargeFastLog: $result"
    }

    if $debug { puts stderr "lifetime pipeline done" }

    if [catch \
          {exec sddsxref ${tmpFile}.life ${tmpFile}.param -pipe=out -leave=* -transfer=param,* -nowarn \
             | sddsxref -pipe ${tmpFile}.data1 -take=DayOfMonth,Time -nowarn \
             | sddsxref -pipe ${tmpFile}.ave \
             -transfer=param,S35DCCTPeak,S35DCCTAverage,*Percent \
             -leave=* -nowarn \
             | sddsxref -pipe ${tmpFile}.integ -take=S35DCCTInteg,S35DCCTGrandInteg -nowarn \
             | sddsprocess -pipe -nowarn \
             "-print=param,StatsLabel,Peak:%.1fmA Total:%.1fmA-h Grand:%.0fmA-h >1mA:%.1f%% Deliv:%.1f%%,S35DCCTPeak,S35DCCTIntegDay,S35DCCTGrandIntegDay,BeamOver1mAPercent,BeamDeliveredPercent" \
             | gzip -c > $pfile} result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSPostprocessChargeFastLog: $result"
    }

    if $debug { puts stderr "Collection pipeline done" }

    if $debug { exec sddsprintout -param=StatsLabel $pfile }
    APSDeleteTmpFileList -ID $tmpID

    if [catch {APSMakeMonthlyChargeFastFile -offset -1} result] {
        return -code error "APSPostprocessChargeFastLog: $result"
    }
    return 
}

proc APSMakeMonthlyChargeFastFile {args} {
    set offset -1
    APSStrictParseArguments {offset}
    if $offset>0 {
        set offset 0
    }

    set dateList [APSOffsetDateInfo -today 1 -offset $offset -dateFormat list -leadingZeros 1 -twoDigitYear 0]
    set year [lindex $dateList 0]
    set month [lindex $dateList 2]
    set fileList [glob -nocomplain ChargeFastProc-${year}-???-${month}??.gz]
    if [llength $fileList] {
        set newFile ChargeFastProc-${year}-${month}.gz
        if [catch \
              {eval exec sddscombine $fileList -pipe=out \
                 | sddscollapse -pipe \
                 | sddsprocess -pipe \
                 -process=S35DCCTPeak,max,S35DCCTPeak \
                 -process=S35DCCTIntegDay,sum,S35DCCTIntegTotal \
                 -process=BeamOver1mAPercent,ave,BeamOver1mAPercent \
                 -process=BeamDeliveredPercent,ave,BeamDeliveredPercent \
                 -process=S35DCCTGrandIntegDay,last,S35DCCTGrandInteg \
                 {{-print=param,MonthlyLabel,Peak %.1fmA   Total %.1fmA-h  Grand %.1fmA-h  >1mA %.0f%%  Deliv %.0%%,S35DCCTPeak,S35DCCTIntegTotal,S35DCCTGrandInteg,BeamOver1mAPercent,BeamDeliveredPercent}} \
                 | gzip -c > $newFile \
             } result] {
            return -code error "APSMakeMonthlyChargeFastFile: $result"
        }
    }
}

proc APSTimeAverageDataLog {args} {
    global apsOAGDataDir
    # The usual variables for specifying where the data is and what it
    # is called.
    set subDirectory ""
    set rootname ""
    # Two lists specifying when and how to average the data.
    # The first list gives boundaries (in days) for file ages at which a new
    # stage of averaging is done.  The averaging interval is in seconds.
    set ageBoundaryList ""
    set averageIntervalList ""
    set errorList ""

    set argcopy "$args"
    if {[APSStrictParseArguments {rootname subDirectory ageBoundaryList averageIntervalList}] || \
          ![string length $subDirectory] || ![string length $rootname] || \
          ![llength $ageBoundaryList] || [llength $ageBoundaryList]!=[llength $averageIntervalList]} {
        return -code error "APSTimeAverageDataLog (1): bad arguments: $argcopy"
    }

    # Check that the lists ageBoundaryList and averageIntervalList
    # contain monotonically increasing values.
    set lastAge [lindex $ageBoundaryList 0]
    set lastInterval [lindex $averageIntervalList 0]
    set index 1
    while {$index<[llength $ageBoundaryList]} {
        if {$lastAge>=[lindex $ageBoundaryList $index]} {
            return -code error "APSTimeAverageDataLog (1.1): bad age boundary list: $ageBoundaryList"
        }
        if {$lastInterval>=[lindex $averageIntervalList $index]} {
            return -code error "APSTimeAverageDataLog (1.2): bad average interval list: $averageIntervalList"
        }
        set lastAge [lindex $ageBoundaryList $index]
        set lastInterval [lindex $averageIntervalList $index]
        incr index
    }

    set oldDir [pwd]
    set dataDir $apsOAGDataDir/$subDirectory
    if [catch {cd $dataDir} result] {
        return -code error "APSTimeAverageDataLog (2): $result"
    }
    if [catch {APSCreateLockFile -fileName APSTimeAverageDataLog.lock} result] {
        return -code error "APSTimeAverageDataLog: $result"
    }

    if [catch {exec timeconvert -now | token -last} timeNow] {
        return -code error "APSTimeAverageDataLog (2.0): $timeNow"
    }

    # Make an SDDS file with a list of all the data files.
    # compute their ages based on the filename.
    set fileList /tmp/[APSTmpString]
    set tmpID [APSUniqueName tmp]
    APSAddToTmpFileList -ID $tmpID -fileList \
      [list $fileList.ascii $fileList]

    if {0} {
        if [catch {exec echo [lsort -decreasing [glob -nocomplain $rootname-????-???-????.gz]] | tr " " "\n" > $fileList.ascii} result] {
            return -code error "APSTimeAverageDataLog (2.1): $result"
        }
    }
    set fid [open $fileList.ascii w]
    puts $fid [join [lsort -decreasing [glob -nocomplain $rootname-????-???-????.gz]] \n]
    close $fid

    if [catch {exec cat $fileList.ascii | wc -l | token -n=1} length] {
        catch {eval file delete -force $filesToDelete}
        return -code error "APSTimeAverageDataLog (2.2): $length"
    }
    if !$length {
        return -code error "APSTimeAverageDataLog (2.3): no files"
    }
    if [catch {exec csv2sdds $fileList.ascii -pipe=out \
                 -columnData=name=Filename,type=string \
                 | sddsprocess -pipe -nowarning -match=column,Filename=,! \
                 -define=parameter,TimeNow,[expr int($timeNow)],type=long,units=s \
                 -edit=column,YearString,Filename,%/$rootname-// \
                 -scan=column,Year,YearString,%ld,type=long \
                 -edit=column,JDayString,Filename,%/$rootname-//Z- \
                 -scan=column,JDay,JDayString,%ld,type=long \
                 -define=column,Hour,0,type=double \
                 | sddstimeconvert -pipe \
                 -epoch=column,CreationTime,year=Year,julian=JDay,hour=Hour \
                 | sddsprocess -pipe=in $fileList -nowarning \
                 "-define=column,DaysOld,TimeNow CreationTime - 3600 / 24 / int,units=days,type=long"} result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSTimeAverageDataLog (3): $result"
    }

    set bpIndex -1
    foreach ageBoundary $ageBoundaryList {
        incr bpIndex

        # Will need to average all those files that either are within the specified age
        # but have no averaging done yet for times longer than this breakpoint

        set averageInterval [lindex $averageIntervalList $bpIndex]
        
        # First make a list of files of the right age.
        # These are files that are older than the age boundary.
        APSAddToTmpFileList -ID $tmpID -fileList \
          [list $fileList.age$bpIndex $fileList.na $fileList.ia $fileList.av $fileList.doav \
             averagingRecord.update$bpIndex averagingRecord.old$bpIndex averagingRecord.new]
        if [catch {exec sddsprocess $fileList $fileList.age$bpIndex -nowarning \
                     -filter=column,DaysOld,$ageBoundary,[expr 40*365]} result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (4): $result"
        }
        if [catch {exec sdds2stream -rows $fileList.age$bpIndex | token -n=1} rows] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (4.1): $rows"
        }
        if !$rows {
            # There are no files older than this boundary.  Could break here instead of
            # continuing.
            continue
        }

        if [file exists averagingRecord.sdds] {
            # Find names of files from list that don't have any averaging recorded.
            # Then, find names of files from list that do have averaging recorded, but keep only
            # those for which no averaging as long as the averaging for this breakpoint has
            # been done.  Note that the averaging interval is in seconds.
            if [catch {exec sddsselect $fileList.age$bpIndex averagingRecord.sdds $fileList.na \
                         -match=Filename -invert -nowarning
                exec sddsxref $fileList.age$bpIndex averagingRecord.sdds -pipe=out \
                         -nowarning -match=Filename -take=* \
                         | sddsprocess -nowarning -pipe=in $fileList.ia \
                         -filter=column,AverageInterval,0,[expr $averageInterval-1]
                exec sddscombine $fileList.na $fileList.ia -merge -pipe=out \
                         | sddssort -pipe=in $fileList.av -column=Filename -unique} result] {
                APSDeleteTmpFileList -ID $tmpID
                return -code error "APSTimeAverageDataLog (5): $result"
            } 
        } else {
            # Since there is no record of any averaging, 
            # will have to check every file of the right age.
            if [catch {exec sddsprocess $fileList.age$bpIndex $fileList.av \
                         -define=column,AverageInterval,0,type=long,units=s } result ] { 
                APSDeleteTmpFileList -ID $tmpID
                return -code error "APSTimeAverageDataLog (6): $result"
            }
        }

        if [catch {exec sdds2stream -rows $fileList.av | token -n=1} rows] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (6.1): $rows"
        }
        if !$rows {
            # All files are properly averaged already for this age boundary.
            continue
        }

        if [catch {APSGetSDDSColumn -fileName $fileList.av -column Filename} filesToCheck] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (9): $filesToCheck"
        }
        # Combine all of the files that are to be checked and look at the
        # AverageInterval values recorded in each file.  Pipe out those that
        # have been averaged over smaller intervals only.
        if [catch {eval exec sddscombine $filesToCheck -delete=column,* -pipe=out \
                     | sddscollapse -pipe \
                     | sddsprocess -pipe -nowarning \
                     "-redefine=column,AverageInterval,AverageInterval" \
                     | sddssort -pipe -column=Filename -column=AverageInterval,incr \
                     | sddsbreak -pipe -change=Filename \
                     | sddsprocess -pipe -clip=0,1,invert -nowarning \
                     | sddscombine -merge -pipe \
                     | sddsprocess -pipe=in $fileList.doav -nowarning \
                     -filter=col,AverageInterval,0,[expr $averageInterval-1]} result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (10): $result\nLooking at $filesToCheck"
        }

        # Get list of all the files that will be averaged.
        if [catch {exec sdds2stream -rows=bare $fileList.doav} result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (11.1): $result\nLooking at $filesToCheck"
        }
        if $result==0 continue
        if [catch {APSGetSDDSColumn -fileName $fileList.doav -column Filename} filesToAverage] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (11.2): $filesToAverage\nLooking at $filesToCheck"
        }
        if ![llength $filesToAverage] {
            # No files to be averaged.  Probably only get here if the average log was deleted
            # for some reason.
            continue
        }
        # Prepare to make averaging record update file.
        set AverageIntervalData ""
        set FilenameData ""
        if [catch {sdds open averagingRecord.update$bpIndex w} fd1] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (11.1): $fd1"
        }
        if [catch {sdds defineColumn $fd1 AverageInterval -type SDDS_LONG -units s
            sdds defineColumn $fd1 Filename -type SDDS_STRING 
            sdds writeLayout $fd1 
            sdds startPage $fd1 [llength $filesToAverage]} result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSTimeAverageDataLog (11.2): $result"
        }
        foreach filename $filesToAverage {
            # Average each of the files using the current time interval.
            # Remove the existing file and gzip the result so that it
            # goes in place of the old file.
            set rootname [file rootname $filename]
            if [catch {eval exec sddsrunstats $filename -pipe=out -nooverlap \
                         -mean=* -window=column=Time,width=$averageInterval \
                         | sddscombine -pipe -merge \
                         | sddsprocess -pipe -nowarning \
                         {"-test=parameter,n_rows 0 >"} \
                         -redefine=parameter,AverageInterval,$averageInterval \
                         | sddsconvert -pipe=in $rootname.av -edit=column,*Mean,e4b4d 
                file rename -force $filename $rootname.old[file extension $filename] 
                file rename -force $rootname.av $rootname
                file delete -force $rootname.old[file extension $filename]
                file delete -force $rootname.gz
                exec gzip $rootname} result] {
                APSDeleteTmpFileList -ID $tmpID
                append errorList "APSTimeAverageDataLog (12): $result\nfile $filename\n"
                continue
            }
            lappend FilenameData $filename
            lappend AverageIntervalData $averageInterval
        }
        if [string length $errorList] {
            global apsDataManagerNormalEmail
            APSSendEMail -address $apsDataManagerNormalEmail \
              -message "$errorList"
        }
        if ![llength $FilenameData] {
            continue
        }

        # Write the records of averaging to the update file.
        if [catch {eval sdds setColumn $fd1 Filename $FilenameData} result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code "APSTimeAverageDataLog (13a): $result\nsdds setColumn $fd1 Filename $FilenameData"
        }
        if [catch {eval sdds setColumn $fd1 AverageInterval $AverageIntervalData} result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code "APSTimeAverageDataLog (13b): $result\nsdds setColumn $fd1 AverageInterval $AverageIntervalData"
        }
        if [catch {sdds writePage $fd1
            sdds close $fd1 } result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code "APSTimeAverageDataLog (13c): $result"
        }
        if {[file exists averagingRecord.sdds]} {
            # Combine the new averaging records with the existing ones, if any, keeping
            # only the largest averaging interval for each file.
            if [catch {exec sddscombine averagingRecord.sdds averagingRecord.update$bpIndex \
                         -pipe=out -merge \
                         | sddssort -pipe -column=Filename -column=AverageInterval,decr \
                         | sddsbreak -pipe -change=Filename \
                         | sddsprocess -pipe "-test=column,i_row 0 ==" \
                         | sddscombine -merge -pipe=in averagingRecord.new 
                exec mv averagingRecord.sdds averagingRecord.old$bpIndex
                exec mv averagingRecord.new averagingRecord.sdds} result] {
                APSDeleteTmpFileList -ID $tmpID
                return -code error "APSTimeAverageDataLog (14): $result"
            } 
        } else {
            # Install this file as the averaging record.
            if [catch \
                  {exec cp averagingRecord.update$bpIndex averagingRecord.sdds} \
                  result] {
                APSDeleteTmpFileList -ID $tmpID
                return -code error "APSTimeAverageDataLog (15): $result"
            }
        }
    }
    APSDeleteTmpFileList -ID $tmpID
    if [string length $errorList] {
        return -code error "$errorList"
    }
}

proc APSCreateLockFile {args} {
    set fileName ""
    APSStrictParseArguments {fileName}

    if ![string length $fileName] {
        return -code error "APSCreateLockFile: no fileName given"
    }
    if [file exists $fileName] {
        if [catch {open $fileName a} fid] {
            return -code error "APSCreateLockFile: unable to open $fileName: $fid"
        }
        if [catch {os lockf $fid F_TLOCK 0} result] {
            return -code error "APSCreateLockFile: $fileName locked by another process: $result"
        }
    } else {
        if [catch {open $fileName w} fid] {
            return -code error "APSCreateLockFile: unable to open $fileName: $fid"
        }
        if [catch {os lockf $fid F_LOCK 0} result] {
            return -code error "APSCreateLockFile: unable to lock $fileName: $result"
        }
        puts $fid "started [clock format [clock seconds]]" 
        flush $fid
        # this is necessary to force NFS to create the file with the
        # name given, but introduces the possibility of another process slipping in
        close $fid
        if [catch {open $fileName a+} fid] {
            return -code error "APSCreateLockFile: unable to reopen $fileName: $fid"
        }
        if [catch {os lockf $fid F_LOCK 0} result] {
            return -code error "APSCreateLockFile: unable to relock $fileName: $result"
        }
    }
    return $fid
}

