#$Log: not supported by cvs2svn $
#Revision 1.26  2011/02/11 15:31:04  shang
#added noUpdate arguments to collect FB bpm and corrector history procedure to avoid press update button repeatedly which is not necessary, but causes errors to FB ioc.
#
#Revision 1.25  2011/01/12 16:24:00  shang
#added pendIOTime to channel access commands.
#
#Revision 1.24  2010/11/09 23:48:25  shang
#added error catches to sdds2stream in collecting FPGA data so that it can continue.
#
#Revision 1.23  2010/11/09 23:30:25  shang
#futher improved FPGA bpm history collection a)collectes the data for the IOCs whoever are ready and runs in parallel
#b)separated the post-processing from collection.
#
#Revision 1.22  2010/11/05 16:23:59  shang
#increased the EPICS_CA_MAX_ARRAY_BYTES to 10000000 for collecting FPGA turn history data.
#
#Revision 1.21  2010/11/02 18:05:43  shang
#removed $tmpfile in collecting FPGA bpm history proc since it does not exist any more.
#
#Revision 1.20  2010/11/02 16:52:15  shang
#modified FPGA bpm history collection procedure: a) added checking the arm states of FPGA bpms, wait until bpms are disarmed b)collect
#FPGA bpms in each IOC in parallel c)post-process the data after collection is done.
#
#Revision 1.19  2009/04/28 15:33:46  shang
#removed the fft processing of FPGA bpm history
#
#Revision 1.18  2009/04/21 20:56:46  shang
#commented out "re-arm the FPGA waveform history" after collecting data.
#
#Revision 1.17  2009/04/20 18:21:27  shang
#moved the "rearm histories for FPGA to the end to set it ready for the next beam dump"
#
#Revision 1.16  2009/04/16 13:46:45  shang
#fixed a typo
#
#Revision 1.15  2009/04/14 14:06:48  shang
#added APSCollectFPGABpmHistory procedure to collect FPGA bpm history.
#
#Revision 1.14  2009/01/31 00:42:44  emery
#Added wild card around Illegal Value because sometimes sddscasr added
#imbedded quotes.
#
#Revision 1.13  2009/01/30 15:06:50  shang
#added one more filter (16) for SelectedMI
#
#Revision 1.12  2009/01/30 00:27:46  shang
#fixed a syntax error
#
#Revision 1.11  2009/01/30 00:08:01  shang
#added an extra test to bpm channel configuration, removing the "Illegal Value" in addition to the "none" values.
#
#Revision 1.10  2006/09/08 16:24:36  soliday
#APSGetFBCorHistory.tcl is not a program so I removed the exec oagwish and
#auto_path commands.
#
#Revision 1.9  2004/05/13 17:11:14  emery
#Added APSCollectNbBpmHistory. Not tested.
#
#Revision 1.8  2003/07/09 21:46:51  shang
#added log messages
#

proc APSCollectFBCorrectorHistory {args} {
    set filename ""
    set noUpdate 0
    APSStrictParseArguments {filename noUpdate}

    # update waveform records before collecting data
    if !$noUpdate {
	if [catch {exec cavput -pend=15 -list=S -range=beg=1,end=39,interval=2 \
		       -list=FB:ReadDataHistoryBO=1} result] {
	    return -code error "APSCollectFBCorrectorHistory: $result"
	}
	exec sleep 2
    }
    
    if [catch {exec sddswmonitor -pendIOTime=100 \
                 /home/helios/oagData/mpsDumps/inputs/FBCorrectorErrors.wmon \
                 $filename} status] {
        return -code error "APSCollectFBCorrectorHistory: $status"
    }
    if [catch {exec cavget -pend=15 -noQuote -list=SRFB:MPS:inp0TimeSI} tripTimeStamp] {
        return -code error "APSCollectFBCorrectorHistory: $tripTimeStamp"
    }
    if [catch {exec cavget -list=SRFB:fsAO -pend=30} rate] {
        return -code error "APSCollectFBCorrectorHistory: $rate"
    }
    set dt [expr 1000.0/$rate]
    if [catch {exec sddsprocess $filename -noWarnings \
                 -print=par,MPSTripTimeStamp,$tripTimeStamp \
                 "-def=col,TimeRelativeToTrip,i_row 127 - $dt *,units=ms"} result] {
        return -code error "APSCollectFBCorrectorHistory: $result"
    }
    catch {file delete ${filename}~}
}

proc APSCollectFBbpmHistory {args} {
    set rootname ""
    set noUpdate 0
    APSStrictParseArguments {rootname noUpdate}
    
    # update waveform records before collecting data
    if !$noUpdate {
	if [catch {exec cavput -pend=15 -list=S -range=beg=1,end=39,interval=2 \
		       -list=FB:ReadDataHistoryBO=1} result] {
	    return -code error "APSCollectFBbpmHistory: $result"
	}
	exec sleep 2
    }
    set tmpfile /tmp/[APSTmpString]
    APSAddToTmpFileList -ID 1 -fileList "$tmpfile $tmpfile.config $tmpfile.1 $tmpfile.2 $tmpfile.x $tmpfile.y $tmpfile.n $tmpfile.config1"
    if [catch {exec sddswmonitor -pendIO=100 \
                 /home/helios/oagData/mpsDumps/inputs/FBbpmErrors.wmon \
                 $tmpfile} status] {
        return -code error "APSCollectFBbpmHistory: $status"
    }
    if [catch {exec cavget -pend=15 -noQuote -list=SRFB:MPS:inp0TimeSI} tripTimeStamp] {
        return -code error "APSCollectFBbpmHistory: $tripTimeStamp"
    }
    if [catch {exec cavget -list=SRFB:fsAO -pend=15} rate] {
        return -code error "APSCollectFBbpmHistory: $rate"
    }
    set dt [expr 1000.0/$rate]
    if [catch {exec sddsprocess $tmpfile $tmpfile.n -noWarnings \
                 -print=par,MPSTripTimeStamp,$tripTimeStamp \
                 "-def=col,TimeRelativeToTrip,i_row 127 - $dt *,units=ms"} result] {
        return -code error "APSCollectFBbpmHistory: $result"
    }
    #rename the history bpm with real bpm names
    #obtain the bpm name from bpm channel configuration
    set columns [exec sddsquery $tmpfile.n -col]
    if [catch {exec sddsmakedataset -column=HistoryName,type=string \
                 -data=[join $columns ,] $tmpfile.1 } result] {
        return -code error $result
    }
    if [catch {exec sddscasr -save -pendIOTime=60 \
                 /home/helios/oagData/mpsDumps/bpmChannelConfig.req $tmpfile.config } result] {
        return -code error "APSCollectFBbpmHistory: $result"
    }
    if [catch {exec sddsprocess $tmpfile.config  \
                 "-match=col,ValueString=None,!"  \
                 "-match=col,ValueString=*Illegal Value*,!" \
                 "-match=col,ValueString=16,!" $tmpfile.config1
        exec  sddsxref  $tmpfile.1 $tmpfile.config1 -match=HistoryName \
                 -take=ValueString $tmpfile.2 -noWarnings } result] {
        return -code error $result
    }
    if [catch {exec sddsprocess $tmpfile.2 $tmpfile.x -match=col,HistoryName=*Horz* 
        exec sddsprocess $tmpfile.2 $tmpfile.y -match=col,HistoryName=*Vert* } result] {
        return -code error "APSCollectFBbpmHistory: $result"
    }
    foreach plane {x y} {
        set historyName [exec sdds2stream $tmpfile.$plane -col=HistoryName]
        set bpmName [exec sdds2stream $tmpfile.$plane -col=ValueString]
        set renameList ""
        foreach col $historyName bpm $bpmName {
            lappend renameList $col=$bpm
        }
        if [catch {exec sddsconvert $tmpfile.n -pipe=out \
                     -retain=col,Index,[join $historyName ,],TimeRelativeToTrip \
                     | sddsconvert -pipe -rename=col,[join $renameList ,] \
                     | sddssortcolumn -pipe=in $rootname.$plane \
                     -sortWith=/home/helios/oagData/mpsDumps/bpmReferenceFile,col=BPMName \
                 } result] {
            return -code error "APSCollectFBbpmHistory: $result"
        }
    }
    catch {file delete $tmpfile $tmpfile.config $tmpfile.1 $tmpfile.2 $tmpfile.x $tmpfile.y $tmpfile.n $tmpfile.config1}
}

proc APSCollectNbBpmHistory {args} {
    set rootname ""
    APSStrictParseArguments {rootname}
    
    # update waveform records before collecting data
    if [catch {exec cavput -pend=15 -list=S -range=beg=1,end=39,interval=2 \
                 -list=FB:ReadNBBPMHistoryBO=1} result] {
        return -code error "APSCollectNbBpmHistory: $result"
    }
    exec sleep 2
    set tmpfile /tmp/[APSTmpString]
    APSAddToTmpFileList -ID 1 -fileList "$tmpfile"
    if [catch {exec sddswmonitor -pendIO=100 \
                 /home/helios/oagData/mpsDumps/inputs/FBNbBpm.wmon \
                 $tmpfile} status] {
        return -code error "APSCollectNbBpmHistory: $status"
    }
    if [catch {exec cavget -pend=15 -noQuote -list=SRFB:MPS:inp0TimeSI} tripTimeStamp] {
        return -code error "APSCollectNbBpmHistory: $tripTimeStamp"
    }
    if [catch {exec cavget -list=SRFB:fsAO -pend=15} rate] {
        return -code error "APSCollectNbBpmHistory: $rate"
    }
    set dt [expr 1000.0/$rate]
    if [catch {exec sddsprocess $tmpfile $rootname -noWarnings \
                 -print=par,MPSTripTimeStamp,$tripTimeStamp \
                 "-def=col,TimeRelativeToTrip,i_row 127 - $dt *,units=ms"} result] {
        return -code error "APSCollectNbBpmHistory: $result"
    }
    catch {file delete $tmpfile}
}

proc APSCollectFPGABpmHistory {args} {
    set turnHistoryName ""
    set slowHistoryName ""
    set directory [pwd]
    set timeout 600
    #set default timing out 10 mintues
    APSStrictParseArguments {turnHistoryName slowHistoryName directory timeout}
    
    set sectorList [exec sdds2stream /home/helios/oagData/sr/FPGAbpm/sectors.sdds -col=Sector]
    #collect turn history
    set turnRoot /tmp/[APSTmpString]turn
    set slowRoot /tmp/[APSTmpString]slow
    foreach sector $sectorList {
	foreach nm {A B} {
	    set subSector S${sector}$nm
	    if [catch {exec replaceText  /home/helios/oagData/mpsDumps/inputs/FPGAbpm${nm}.wmon.template \
                         ${turnRoot}-${subSector}.wmon \
                         -orig=<sector>,<type>,<waveformLength> -repl=$sector,turnHistory,262144
		exec replaceText  /home/helios/oagData/mpsDumps/inputs/FPGAbpm${nm}.wmon.template \
                         ${slowRoot}-${subSector}.wmon \
                         -orig=<sector>,<type>,<waveformLength> -repl=$sector,motion,2048
		exec replaceText /home/helios/oagData/mpsDumps/inputs/FPGAbpm.mon.template \
                         ${turnRoot}-${subSector}.mon -orig=<subSector> -repl=$subSector
                file copy -force ${turnRoot}-${subSector}.mon ${slowRoot}-${subSector}.mon } result] {
		return -code error "Error in generating FPGA history mon file: $result"	
	    }
	    catch {exec collectFPGABpmHistory -cleanup 1 \
                     -subSector $subSector -monFile ${turnRoot}-${subSector}.wmon \
                     -scalarFile ${turnRoot}-${subSector}.mon -turnHistory turn  -timeout $timeout \
                     -outputFile $directory/FPGA-${subSector}.turnHistory.sdds  &}
	    after 1000
	    catch {exec collectFPGABpmHistory -cleanup 1 \
                     -subSector $subSector -monFile ${slowRoot}-${subSector}.wmon \
                     -scalarFile ${slowRoot}-${subSector}.mon -turnHistory motion -timeout $timeout \
                     -outputFile $directory/FPGA-${subSector}.slowHistory.sdds  &}
	    after 1000
        }
    }
}

proc APSProcessFPGABpmHistoryData {args} {
    set dirName ""
    APSStrictParseArguments {dirName}
    
    set turnFiles [glob -nocomplain $dirName/FPGA-*.turnHistory.sdds]
    set revolutionTime 3.682547610987599e-06
    foreach file $turnFiles {
        # put the -ifis sddsprocess first.
        # newer data will have the parameters HistoryLength PostTriggerLength defined
        file delete -force ${file}.gz
        if [catch {exec sddsprocess $file ${file}.gz \
                       -ifis=para,PostTriggerLength \
                       -noWarnings "-redefine=col,Time,Index $revolutionTime *,units=s" \
                       "-redefine=col,%s,%s 1000. /,select=*sum,units=mV" \
                       "-redefine=col,%s,%s,selection=*:x,units=mm" \
                       "-redefine=col,%s,%s,selection=*:y,units=mm" \
                       "-redef=col,TimeRelativeToTrip,Index HistoryLength PostTriggerLength - - $revolutionTime * 1000 *,units=ms" } result] {
            return -code error "Error1 in processing turn history data: $result"
        }
        if [catch {exec sddsprocess $file ${file}.gz \
                       -ifnot=para,PostTriggerLength \
                       -redef=para,PostTriggerLength,131072 \
                       -redef=para,HistoryLength,262144 \
                       -noWarnings "-redefine=col,Time,Index $revolutionTime *,units=s" \
                       "-redefine=col,%s,%s 1000. /,select=*sum,units=mV" \
                       "-redefine=col,%s,%s,selection=*:x,units=mm" \
                       "-redefine=col,%s,%s,selection=*:y,units=mm" \
                       "-redef=col,TimeRelativeToTrip,Index HistoryLength PostTriggerLength - - $revolutionTime * 1000 *,units=ms" } result] {
            return -code error "Error2 in processing turn history data: $result"
        }
        if {![file exists ${file}.gz]} {
            puts stderr "Did not process $file"
            continue
        }

        if [regexp {A.turn} $file] {
            set bpmList {A:P2 A:P3 A:P4}
        } else {
            set bpmList {B:P2 B:P3 B:P4 B:P5}
        }
        set sector [scan [file tail $file] FPGA-S%ld]
        foreach bpm $bpmList {
            foreach plane {x y} {
                if [catch {exec sddsconvert ${file}.gz \
                             -retain=col,Time,S${sector}${bpm}*${plane}* \
                             -pipe=out \
                             | sddsprocess -pipe -sparse=2 -filter=col,S${sector}${bpm}:${plane}sum,0,1 -nowarnings \
                             | sddsfft -pipe=in  $dirName/FPGA-S${sector}${bpm}.turnHistory.$plane.fft -truncate -nowarnings \
                             -suppressAverage -psdOutput,integrated \
                             -col=Time,S${sector}${bpm}:${plane} } result] {
                    file delete -force ${file}.gz
                    return -code error "Error3 in doing fft for turn history: $result"
                }
            }
        }
        file delete -force $file
    }
    set slowFiles [glob -nocomplain $dirName/FPGA-*.slowHistory.sdds]
    
    foreach file $slowFiles {
        #process slow beam history files. 1024 is the usual post-trigger length
        # put the -ifis sddsprocess first.
        file delete -force ${file}.gz
        if [catch {exec sddsprocess $file ${file}.gz \
                       -ifis=para,PostTriggerLength \
                       -redef=para,Rate,100,units=Hz \
                       -noWarnings "-redefine=col,Time,Index  0.01 *,units=s" \
                       "-redefine=col,%s,%s,select=*sum,units=mV" \
                       "-redefine=col,%s,%s,select=*:x,units=mm" \
                       "-redefine=col,%s,%s,select=*:y,units=mm" \
                       "-redef=col,TimeRelativeToTrip,Index HistoryLength PostTriggerLength - - 1000.0 Rate / *,units=ms" } result] {
            return -code error "Error4 in processing slow history: $result"
        }
        if [catch {exec sddsprocess $file ${file}.gz \
                       -ifnot=para,PostTriggerLength \
                       -redef=para,Rate,100,units=Hz \
                       -redef=para,PostTriggerLength,1024 \
                       -redef=para,HistoryLength,2048 \
                       -noWarnings "-redefine=col,Time,Index  0.01 *,units=s" \
                       "-redefine=col,%s,%s,select=*sum,units=mV" \
                       "-redefine=col,%s,%s,select=*:x,units=mm" \
                       "-redefine=col,%s,%s,select=*:y,units=mm" \
                       "-redef=col,TimeRelativeToTrip,Index HistoryLength PostTriggerLength - - 1000.0 Rate / *,units=ms" } result] {
            return -code error "Error5 in processing slow history: $result"
        }
        if {![file exists ${file}.gz]} {
            puts stderr "Did not process $file"
            continue
        }

        if [regexp {A.slow} $file] {
            set bpmList {A:P2 A:P3 A:P4}
        } else {
            set bpmList {B:P2 B:P3 B:P4 B:P5}
        }
        set sector [scan [file tail $file] FPGA-S%ld]
        foreach bpm $bpmList {
            foreach plane {x y} {
                if [catch {exec sddsconvert $file.gz \
                             -retain=col,Time,S${sector}${bpm}*${plane}* \
                             -pipe=out \
                             | sddsprocess -pipe -sparse=2 -filter=col,S${sector}${bpm}:${plane}sum,0,1000 -nowarnings \
                             | sddsfft -pipe=in  $dirName/FPGA-S${sector}${bpm}.slowHistory.$plane.fft -truncate \
                             -suppressAverage -psdOutput,integrated -nowarnings \
                             -col=Time,S${sector}${bpm}:${plane} } result] {
                    file delete -force ${file}.gz
                    return -code error "Error6 in doing fft for slow history: $result"
                }
            }
        }
        file delete -force $file
    }
    return
}
