#!/bin/sh  
# \
  exec oagtclsh "$0" "$@"

set auto_path [linsert $auto_path 0  /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]

proc ConvertPVstoReadbackNames {pvList} {
    puts "Converting PV names to data logger readback names"

    if {[catch {exec sddsprocess /home/helios/oagData/dataLoggerConfig/timeSeries.config -pipe=out \
                  "-print=column,MonitorFile,/home/helios/oagData/%s/%s.mon,subDirectory,rootname" \
                  "-print=column,LoggingConditions,%s,extraArguments" \
                  "-match=column,GroupName=*" \
                  "-match=column,monitorProgram=sddsglitchlogger,!,monitorProgram=sddspvaglitchlogger,!,&" \
                  "-filter=column,doRun,1,1,doOnePvPerFileRun,1,1,|" | \
                  sdds2stream -pipe=in -col=MonitorFile} monitorFileList]} {
        puts stderr "Error: $monitorFileList"
        exit 1
    }

    set nameList ""
    foreach monitorFile $monitorFileList {
        if {[catch {exec sddsquery $monitorFile -col} results]} {
            puts stderr "$results\n$monitorFile"
            exit 1
        }
        if {[lsearch -exact $results "ControlName"] == -1} {
            continue
        }
        if {[lsearch -exact $results "ReadbackName"] == -1} {
            if {[catch {exec sdds2stream $monitorFile -col=ControlName,ControlName} results]} {
                puts stderr "$results\n$monitorFile"
                exit 1
            }
        } else {
            if {[catch {exec sdds2stream $monitorFile -col=ControlName,ReadbackName} results]} {
                puts stderr "$results\n$monitorFile"
                exit 1
            }
        }
        append nameList "[split $results \n] "
    }
    global pvReadbacks
    foreach pv $pvList {
        if {[string range $pv end-3 end] == ".VAL"} {
            set pv1 [string range $pv 0 end-4]
        } else {
            set pv1 $pv
        }
        set results [lsearch -index 0 -all -inline $nameList $pv1]
        set rbNameList ""
        foreach result $results {
            lappend rbNameList [lindex $result 1]
        }
        set rbNameList [lsort -unique $rbNameList]
        set pvReadbacks($pv) $rbNameList
    }
}

proc FindPVDataFiles {args} {
    set ReadbackName ""
    set StartTime [clock scan "Jan 01 00:00:00 1990"]
    set EndTime [clock seconds]
    set startDateList ""
    set endDateList ""
    set SampleIntervals "1 2 4 8 16 32 64 128 256"
    APSStrictParseArguments {ReadbackName StartTime EndTime SampleIntervals startDateList endDateList}

    if {[llength $startDateList]} {
        set StartTime [clock scan "[lindex $startDateList 2]/[lindex $startDateList 3]/[lindex $startDateList 0]"]
    }
    if {[llength $endDateList]} {
        set EndTime [clock scan "23:59:59 [lindex $endDateList 2]/[lindex $endDateList 3]/[lindex $endDateList 0]"]
    }
    
    set fileList ""
    set ReadbackName [join [split $ReadbackName / ] + ]
    set year [clock format $StartTime -format %Y]
    set month [clock format $StartTime -format %m]
    set endyear [clock format $EndTime -format %Y]
    set endmonth [clock format $EndTime -format %m]
    while {($year < $endyear) || (($year == $endyear) && ($month <= $endmonth))} {
        append fileList "[lsort [glob -nocomplain /home/helios/oagData/logging/Variable/[string index ${ReadbackName} 0]/${ReadbackName}/log-${year}-${month}.{gz,xz}]] "
        append fileList "[lsort [glob -nocomplain /home/helios/oagData/logging/\{[join $SampleIntervals ,]\}/${ReadbackName}/log-${year}-${month}.{gz,xz,????}]] "
        if {$month == 12} {
            set month 01
            incr year
        } else {
            scan $month %d month
            incr month
            set month [format %02d $month]
        }
    }
    return $fileList
}

set input ""
set pvnames ""
set rbnames ""
set rbnames ""
set output ""
set timestart ""
set timeend ""

set args $argv
if {[APSStrictParseArguments {input output timestart timeend pvnames rbnames}]} {
    puts stderr "usage: extractSDDSlogs (-pvnames <list> | -rbnames <list> | -input <filename>) -output <fileroot> -timestart <date/timestring or epoch time> -timeend <date/timestring or epoch time>"
    exit 1
}
if {(([string length $pvnames] == 0) && ([string length $rbnames] == 0) && ([string length $input] == 0)) || ([string length $output] == 0) || ([string length $timestart] == 0) || ([string length $timeend] == 0)} {
    puts stderr "usage: extractSDDSlogs (-pvnames <list> | -rbnames <list> | -input <filename>) -output <fileroot> -timestart <date/timestring or epoch time> -timeend <date/timestring or epoch time>"
    exit 1
}

if {[catch {clock format $timestart} results]} {
    set StartTime [clock scan $timestart]
} else {
    set StartTime $timestart
}
if {[catch {clock format $timeend} results]} {
    set EndTime [clock scan $timeend]
} else {
    set EndTime $timeend
}

if {[string length $input] != 0} {
    if {[catch {exec sddscheck $input -printerror} result]} {
        set fid [open $input r]
        set pvList [read $fid]
        close $fid
    } else {
        if {[catch {exec sdds2stream $input -col=ControlName} pvList]} {
            puts stderr "Error: expected either sdds file with ControlName column or ascii file with list of PV names"
            puts stderr "$pvList"
            exit 1
        }
    }
    ConvertPVstoReadbackNames $pvList
} elseif {[string length $pvnames] != 0} {
    set pvList [join [split $pvnames ,] " "]
    ConvertPVstoReadbackNames $pvList
} else {
    set pvList [join [split $rbnames ,] " "]
    foreach pv $pvList { 
        set pvReadbacks($pv) $pv
    }
}

package require Thread
if {[catch {exec grep processor /proc/cpuinfo | wc -l} threads]} {
    set threads 8
}
set pool [tpool::create -maxworkers $threads -initcmd {
proc Proc1 {input output start end} {
    if {[catch {exec sddsprocess $input $output -filter=column,Time,$start,$end -nowarn } results]} {
        after 1000
        if {[catch {eval exec sddsprocess $input $output -filter=column,Time,$start,$end -nowarn } results]} {
            return -code error "$results"
        }
    }
    if {[catch {exec sdds2stream $output -rows=bare} rows]} {
        return -code error "$rows"
    }
    return $rows
}
proc Proc2 {inputs output start end} {
    if {[catch {eval exec sddscombine $inputs -pipe=out -merge | \
                  sddsprocess -pipe -filter=column,Time,$start,$end -nowarn | \
                  sddssort -pipe=in $output -col=Time -unique} results]} {
        after 1000
        if {[catch {eval exec sddscombine $inputs -pipe=out -merge | \
                      sddsprocess -pipe -filter=column,Time,$start,$end -nowarn | \
                      sddssort -pipe=in $output -col=Time -unique} results]} {
            return -code error "$results"
        }
    }
    if {[catch {exec sdds2stream $output -rows=bare} rows]} {
        return -code error "$rows"
    }
    return $rows
}
}]


foreach sampleInterval "1 2 4 8 16 32 64 128 256" {
    puts "Working on data logs with a sample interval of $sampleInterval seconds(s)"
    set index 0
    set DataFileList ""
    set tmpFile ${output}.[APSTmpString]
    set i 1
    unset -nocomplain files
    unset -nocomplain files2
    foreach pv $pvList {
        foreach rbName $pvReadbacks($pv) {
            set files($rbName) [FindPVDataFiles -ReadbackName $rbName -StartTime $StartTime -EndTime $EndTime -SampleIntervals $sampleInterval]
        }
        incr i
    }
    set sec2 [clock seconds]
    set i 1
    set jobs ""
    foreach pv $pvList {
        foreach rbName $pvReadbacks($pv) {
            if {[llength $files($rbName)] > 1} {
                set job [tpool::post $pool [list Proc2 $files($rbName) ${tmpFile}$index $StartTime $EndTime]]
                lappend jobs $job
                lappend DataFileList ${tmpFile}$index
                incr index
            } elseif {[llength $files($rbName)] == 1} {
                set job [tpool::post $pool [list Proc1 [string trim $files($rbName)] ${tmpFile}$index $StartTime $EndTime]]
                lappend jobs $job
                lappend DataFileList ${tmpFile}$index
                incr index  
            }
        }
        incr i
    }
    if {[llength $jobs]} {
        tpool::wait $pool $jobs waitingJobs
        while {[llength $waitingJobs] > 0} {
            tpool::wait $pool $waitingJobs waitingJobs
        }
        foreach job $jobs f $DataFileList {
            if {[catch {tpool::get $pool $job} rows]} {
                puts "Error: $rows"
                exit
            }
            lappend files2($rows) $f
        }
    }
    set sec3 [clock seconds]
    puts "Operation took [expr $sec3 - $sec2] seconds"

    if {[llength $DataFileList] == 0} { 
        puts "No data found for sample interval $sampleInterval second(s)\n"
        continue
    }
    set tmpFiles $DataFileList
    foreach r [lsort -integer -decreasing [array names files2]] {
        set first 1
        while {[llength $files2($r)] >= 1} {
            set dfl [lrange $files2($r) 0 1000]
            set files2($r) [lrange $files2($r) 1001 end]
            if {$first} {
                set lastFile ""
                set first 0
            } else {
                set lastFile ${tmpFile}$index
            }
            incr index
            lappend tmpFiles ${tmpFile}$index
            if {([llength $dfl] == 1) && ($lastFile == "")} {
                if {[catch {exec sddsconvert $dfl ${tmpFile}$index -delete=column,CAerrors} results]} {
                    puts "$results"
                    exit
                }
            } else {
                if {[catch {eval exec sddsxref $lastFile $dfl ${tmpFile}$index -equate=Time "-take=*" -leave=CAerrors -nowarn} results]} {
                    puts "$results"
                    exit
                }
            }
        }
        if {[catch {exec sddssort ${tmpFile}$index ${output}.${sampleInterval}.${r}.sdds -col=Time -unique} results]} {
            puts "$results"
            exit
        }
        puts "Data saved to ${output}.${sampleInterval}.${r}.sdds ($sampleInterval second interval, $r rows)"
    }
    puts ""
    eval file delete $tmpFiles ${output}.time.sdds
}
tpool::release $pool
