

# make plots for orbit glitch data display
# shows a movie of x and y orbit changes.
#NOT USED
proc APSSROrbitGlitchPlots {args} {
    return [eval APSSRFindOrbitGlitchSource $args -findSources 0 -plot 1]
}

#NOT USED
proc APSSRFindOrbitGlitchSource {args} {
    return [eval APSSRFindOrbitSource $args -glitch 1]
}

#Used by SRMeasureSlowMotion
#However last known use was in 2000. It is now 2023. So......
proc APSSRFindOrbitSource {args} {
    set input ""
    set output /tmp/[APSTmpString]
    set deleteInput 0
    set display 0
    set plot 0
    set overlapPlots 1
    set findSources 1
    set title ""
    set statusCallback APSNoOp
    set glitch 0
    set selectionLabel ""
    set orbitsToAverage 0
    set lattice default
    set autoscale 0
    if {[APSStrictParseArguments {input output display title deleteInput plot \
                                    findSources statusCallback overlapPlots \
                                    glitch selectionLabel orbitsToAverage autoscale \
                                lattice}] || \
          ![string length $input]} {
        return -code error "APSSRFindOrbitSource: bad arguments"
    }
    
    if ![file exists $input] {
        return -code error "APSSRFindOrbitSource: not found: $input"
    }

    set tmpID [APSUniqueName tmpID]
    APSAddToTmpFileList -ID $tmpID -fileList $output
    set tmpRoot /tmp/[APSTmpString]
    APSAddToTmpFileList -ID $tmpID \
      -fileList [list $tmpRoot.1 $tmpRoot.2 $tmpRoot.3 $tmpRoot.4.x $tmpRoot.4.y \
                   $tmpRoot.5.x $tmpRoot.5.y $tmpRoot.1c $tmpRoot.x.sdds $tmpRoot.y.sdds \
                   $tmpRoot.x.count $tmpRoot.y.count $tmpRoot.h $tmpRoot.v $tmpRoot.x $tmpRoot.y \
                   $tmpRoot.x.printout $tmpRoot.y.printout ]
#    set SROrbitGlitchDir /home/helios/oagData/glitchLogs/SROrbit
    set SROrbitGlitchDir /home/helios/oagData/sr/lattices/$lattice
    set xrefDir /home/helios/oagData/sr/lattices/scripts

    if $glitch {
        # Find the first reading for each BPM, and subtract from all readings to
        # get delta orbits.
        set mode copy
        set discard 1
    } else {
        # Find the average reading for each BPM and subtract from all readings to
        # get delta orbits.
        set mode mean
        set discard 0
    }
    # Collect data for x and y into a single column each, one page per orbit.
    # Xref in the BPM position and sort by it.
    # Compute statistics of orbit changes and make a label for use in plots.
    eval $statusCallback "Processing orbit data to find changes"
    if {$orbitsToAverage>1} {
        set title "$title  averaged over $orbitsToAverage orbits"
        if [catch {exec sddsrunstats $input -pipe=out \
                     -points=$orbitsToAverage -mean=*x,*y,S35DCCT,Time \
                     | sddsconvert -pipe=in $tmpRoot.1 \
                     -retain=column,*Mean -editnames=column,*Mean,%/Mean//} result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSSRFindOrbitSource: $result"
        }
        set input $tmpRoot.1
    }
        
    set hMatrix $tmpRoot.h
    set vMatrix $tmpRoot.v
    if [catch {exec sddscollect $input $tmpRoot.2 \
                 -collect=suffix=:msAve:x,column=x -collect=suffix=:msAve:y,column=y 
        exec sddsselect $SROrbitGlitchDir/sr.h1rm $tmpRoot.2 -match=BPMName=Rootname \
                 $hMatrix
        exec sddsselect $SROrbitGlitchDir/sr.v1rm $tmpRoot.2 -match=BPMName=Rootname \
                 $vMatrix
        exec sddsenvelope $tmpRoot.2 -pipe=out -$mode=x,y -copy=Rootname \
                 | sddsconvert -pipe=in $tmpRoot.3 -editNames=column,*Mean,%/Mean//
        exec sddschanges $tmpRoot.2 -pipe=out -baseline=$tmpRoot.3 -changesIn=x,y -copy=Rootname \
                 | sddsxref -pipe $xrefDir/SRBPMPosition.xref -nowarning \
                 -match=Rootname=BPMName -reuse=row,page -take=s \
                 | sddssort -pipe -column=s \
                 | sddstimeconvert -pipe -breakDown=parameter,Time,text=DataTimeStamp \
                 | sddsprocess -pipe -nowarning \
                 "-test=parameter,i_page $discard == !" \
                 -process=ChangeIn*,mad,%sMAD \
                 "-define=parameter,OrbitNumber,i_page $discard -,type=long" \
                 "-print=parameter,Label1,%s  %.2lf mA,DataTimeStamp,S35DCCT" \
                 | tee $tmpRoot.1 \
                 | sddscollapse -pipe=in $tmpRoot.1c } result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSSRFindOrbitSource: $result"
    }

    # Get the MAD values for x and y orbits.  These should indicate
    # noise levels.
    if [catch \
          {exec sddsprocess $tmpRoot.1c -pipe=out -process=ChangeIn*MAD,ave,%sMean \
             | sdds2stream -pipe=in -param=ChangeInxMADMean,ChangeInyMADMean} MADList] {
        return -code error "APSSRFindOrbitSource: $MADList"
    }
    set xMAD [lindex $MADList 0]
    set yMAD [lindex $MADList 1]

    # Smooth the orbits by despiking, with threshold of 3*MAD.
    if [catch \
          {exec sddssmooth $tmpRoot.1 -pipe=out -new -column=ChangeInx \
             -pass=0 -despike=neighbor=2,aver=2,pass=1,threshold=[expr 3*$xMAD] \
             | sddssmooth -pipe -new -column=ChangeIny \
             -pass=0 -despike=neighbor=6,aver=2,pass=1,threshold=[expr 3*$yMAD] \
             | sddsxref -nowarning -pipe=in $tmpRoot.1 -leave=* -transfer=param,* $tmpRoot.2 \
         } result] {
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSSRFindOrbitSource: $result"
    }

    if $plot {
        if $autoscale {
            exec sddsplot $tmpRoot.2 -graph=line,vary \
              -split=page -groupby=page,request -separate=request -layout=1,2 \
              -legend=edit=%/ChangeIn/ \
              -column=s,(ChangeInx,ChangeInxSmoothed) -topline=@Label1 -title= \
              -column=s,(ChangeIny,ChangeInySmoothed) -topline= "-title=$title" &
        } else {
            exec sddsplot $tmpRoot.2 -limit=ymax=0.2,ymin=-0.2,autoscaling -graph=line,vary \
              -split=page -groupby=page,request -separate=request -layout=1,2 \
              -legend=edit=%/ChangeIn/ \
              -column=s,(ChangeInx,ChangeInxSmoothed) -topline=@Label1 -title= \
              -column=s,(ChangeIny,ChangeInySmoothed) -topline= "-title=$title" &
        }
    }
    if !$findSources {
        after 20000 "APSDeleteTmpFileList -ID $tmpID"
        return
    }

    if $glitch {
        # Look at all but the first 5 orbits to see what the sources are.
        # The glitch logger is set with a pre-buffer of 10,
        # but I allow 5 of those to be analyzed as the precursor to the 
        # trigger.
        if {[catch {exec sdds2stream -rows $tmpRoot.2 | wc -l} orbits]} {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSSRFindOrbitSource: $orbits"
        }
        if $orbits<5 {
            set pageOffset [expr int($orbits/2)]
        } else {
            set pageOffset 5
        }
        if [catch \
              { exec sddsconvert $tmpRoot.2 $tmpRoot.3 -fromPage=[expr $pageOffset+1] } \
              result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSSRFindOrbitSource: $result"
        }
        set input $tmpRoot.3
    } else {
        # look at all of the orbits
        set pageOffset 0
        set input $tmpRoot.2
    }

    set HV(x) h1
    set HV(y) v1
    set matrix(x) $hMatrix
    set matrix(y) $vMatrix
    foreach plane {x y} {
        if [catch \
              {exec sddsconvert $input -pipe=out -retain=column,ChangeIn${plane}Smoothed \
                 | sddstranspose -pipe \
                 | sddsmatrixmult -reuse -pipe $matrix($plane) \
                 | sddsconvert -pipe -delete=column,s \
                 | sddstranspose -pipe -oldColumnNames=CorrectorName -root=Overlap \
                 | sddsprocess -pipe "-redefine=column,Overlap,Overlap abs" \
                 | tee $tmpRoot.4.$plane \
                 | sddssort -pipe -column=Overlap,decr \
                 | sddsprocess -pipe -clip=1,0,invert \
                 "-define=column,OrbitNumber,i_page $pageOffset +,type=long" \
                 | sddscombine -merge -pipe \
                 | sddsxref -nowarning -pipe $tmpRoot.1c -nowarning -equate=OrbitNumber -take=DataTimeStamp \
                 | sddssort -pipe -column=Overlap,decr -column=OrbitNumber,decr \
                 | tee $tmpRoot.$plane.sdds \
                 | sddsprintout -pipe=in $tmpRoot.$plane \
                 "-title=$title\n$selectionLabel" \
                 -column=CorrectorName,label=Corrector \
                 -column=Overlap -column=OrbitNumber -column=DataTimeStamp
                  exec sddssort $tmpRoot.$plane.sdds -pipe=out \
                 -column=CorrectorName -column=Overlap,decr \
                 | sddsbreak -pipe -change=CorrectorName \
                 | sddsprocess -pipe \
                 -process=CorrectorName,first,CorrectorName \
                 -process=Overlap,count,NumberOfOccurences \
                 -process=Overlap,max,OverlapMax \
                 -process=Overlap,median,OverlapMedian \
                 | sddscollapse -pipe \
                 | sddssort -pipe -column=NumberOfOccurences,decr \
                 | sddsprintout -pipe=in $tmpRoot.$plane.count \
                 -column=CorrectorName,label=Corrector \
                 -column=NumberOfOccurences,format=%.0f \
                 "-column=OverlapMax,label=Max. Overlap" \
                 "-column=OverlapMedian,label=Median Overlap" \
                 "-title=$title\n$selectionLabel"
                  exec cat $tmpRoot.$plane.count $tmpRoot.$plane > $tmpRoot.$plane.printout
               } result] {
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSSRFindOrbitSource: $result"
        }
        if $display {
            APSFileDisplayWindow [APSUniqueName .] \
              -fileName $tmpRoot.$plane.printout -deleteOnClose 1 \
              -width 80 -comment "Orbit glitch source analysis\nTitle: $title"
        } 
        if {$plot && $overlapPlots} {
            if [catch {exec sddsxref -nowarning $tmpRoot.4.$plane \
                         $xrefDir/SRCorrPosition.xref \
                         $tmpRoot.5.$plane -nowarning -match=CorrectorName=ElementName -take=s } result] {
                APSDeleteTmpFileList -ID $tmpID
                return -code error "APSSRFindOrbitSource: $result"
            }
            exec sddsplot -column=s,Overlap $tmpRoot.5.$plane -split=page -graph=line,vary \
              "-title=Overlap for each orbit in the $plane plane" &
        }
    }
    after 20000 "APSDeleteTmpFileList -ID $tmpID"
}

#NOT USED
proc APSSRCorrectorGlitchPlots {args} {
    set input ""
    set output /tmp/[APSTmpString]
    set deleteInput 0
    set title ""
    set selectionLabel ""
    set statusCallback APSNoOp
    set plot 0
    set display 0

    set argv [join $args ,]
    if {[APSStrictParseArguments {input output deleteInput plot title \
                                    statusCallback display deleteInput selectionLabel}] || \
          ![string length $input]} {
        return -code error "APSSRCorrectorGlitchPlots: bad arguments: $argv"
    }
    
    if ![file exists $input] {
        return -code error "APSSRCorrectorGlitchPlots: not found: $input"
    }

    set tmpID [APSUniqueName tmpID]
    set tmpRoot /tmp/[APSTmpString]
    APSAddToTmpFileList -ID $tmpID -fileList [list $output $tmpRoot.1 $tmpRoot.2 $tmpRoot.3 $tmpRoot.4]
    set xrefDir /home/helios/oagData/sr/lattices/scripts

    # Collect data for H and V into a single column each, one page per orbit.
    # Xref in the corrector position and sort by it.
    # Compute statistics of changes and make a label for use in plots.
    eval $statusCallback {"Processing data to find changes"}
        
    if [catch {exec sddscollect $input -pipe=out \
                 -collect=match=*H*CurrentAI,column=HCurrentAI,edit=%/:H/:HV/%/:CurrentAI// \
                 -collect=match=*V*CurrentAI,column=VCurrentAI,edit=%/:H/:HV/%/:CurrentAI// \
                 | sddschanges -pipe -changesIn=HCurrentAI,VCurrentAI -copy=Rootname \
                 | sddsprocess -pipe -edit=column,HCorrName,Rootname,%/:HV/:H/ \
                 | sddsxref -nowarning -pipe $xrefDir/SRCorrPosition.xref \
                 -match=HCorrName=ElementName -reuse=row,page -take=Sector \
                 | sddssort -pipe -column=Sector \
                 | sddstimeconvert -pipe -breakDown=parameter,Time,text=DataTimeStamp \
                 | sddsprocess -pipe=in $output \
                 -process=*CurrentAI,largest,%sWorst \
                 -process=*CurrentAI,largest,%sWorstSupply,functionOf=Rootname,position \
                 -reedit=parameter,ChangeInHCurrentAIWorstSupply,%/V// \
                 -reedit=parameter,ChangeInVCurrentAIWorstSupply,%/H// \
                 "-print=parameter,Label1,%s  %.2lf mA  biggest: %s %s,DataTimeStamp,S35DCCT,ChangeInHCurrentAIWorstSupply,ChangeInVCurrentAIWorstSupply"} result] {
        eval $statusCallback {"APSSRCorrectorGlitchPlots: $result"}
        APSDeleteTmpFileList -ID $tmpID
        return -code error "APSSRCorrectorGlitchPlots: $result"
    }

    if $display {
        if [catch {exec sddscollapse $output -pipe=out \
                     | tee $tmpRoot.2 \
                     | sddsprintout -pipe=in $tmpRoot.1 \
                     "-title=$title\n$selectionLabel" \
                     -column=DataTimeStamp,label=Time,format=%26s -column=S35DCCT,format=%5.2f  \
                     "-column=ChangeInHCurrentAIWorstSupply,label=Worst H Supply" \
                     "-column=ChangeInHCurrentAIWorst,label=Worst H Value,format=%5.2f" \
                     "-column=ChangeInVCurrentAIWorstSupply,label=Worst V Supply" \
                     "-column=ChangeInVCurrentAIWorst,label=Worst V Value,format=%5.2f"} result] {
            eval $statusCallback {"APSSRCorrectorGlitchPlots: $result"}
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSSRCorrectorGlitchPlots: $result"
        }
        APSFileDisplayWindow [APSUniqueName .] \
          -fileName $tmpRoot.1 -sddsExportableFile $tmpRoot.2 -width 105 -deleteOnClose 1

        if [catch {exec sddsbreak $input -rowlimit=1 -pipe=out \
                     | sddschanges -pipe \
                     -changesIn=*CurrentAI -copy=Time,S35DCCT \
                     | sddsconvert -pipe -editNames=column,ChangeIn*,%/ChangeIn// \
                     | sddscombine -merge -pipe \
                     | sddsprocess -pipe -process=*CurrentAI,largest,%sLargest \
                     -process=*CurrentAI,drange,%sDRange \
                     -process=*CurrentAI,rms,%sRMS -process=*CurrentAI,qrange,%sQRange \
                     | sddscollapse -pipe \
                     | sddscollect -pipe -collect=suffix=:CurrentAILargest,column=ChangeLargest \
                     -collect=suffix=:CurrentAIDRange,column=ChangeDecileRange \
                     -collect=suffix=:CurrentAIQRange,column=ChangeQuartileRange \
                     -collect=suffix=:CurrentAIRMS,column=ChangeRMS \
                     | sddssort -pipe -column=ChangeLargest,decr \
                     | tee $tmpRoot.3 \
                     | sddsprintout -pipe=in $tmpRoot.4 "-title=$title\n$selectionLabel" \
                     -column=Rootname,label=Supply \
                     -column=(ChangeLargest,ChangeDecileRange,ChangeQuartileRange,ChangeRMS) } result] {
            eval $statusCallback {"APSSRCorrectorGlitchPlots: $result"}
            APSDeleteTmpFileList -ID $tmpID
            return -code error "APSSRCorrectorGlitchPlots: $result"
        }
        APSFileDisplayWindow [APSUniqueName .] \
          -fileName $tmpRoot.4 -sddsExportableFile $tmpRoot.3 -width 105 -deleteOnClose 1
    }

    if $plot {
        eval $statusCallback {"Plotting results."}
        exec sddsplot $output -graph=impulse,vary -ticks=xspac=5 -subticks=xdiv=5 \
          -split=page -groupby=page,tag,request -separate=tag -layout=1,2 \
          -column=Sector,ChangeInHCurrentAI -same -topline=@Label1 "-title=$selectionLabel" -tag=0 \
          -match=col,Rootname=*HV3,\! \
          -column=Sector,ChangeInHCurrentAI -same -topline=@Label1 "-title=$selectionLabel" -tag=0 \
          -match=col,Rootname=*HV3 \
          -column=Sector,ChangeInVCurrentAI -same -topline= "-title=$title" -tag=1 \
          -match=col,Rootname=*HV3,\! \
          -column=Sector,ChangeInVCurrentAI -same -topline= "-title=$title" -tag=1 \
          -match=col,Rootname=*HV3 \
          &
    }
    after 20000 "APSDeleteTmpFileList -ID $tmpID"
}

#NOT USED
proc APSSRFindOrbitGlitchSourceSVD {args} {
    set input ""
    set title ""
    set deleteInput 0
    set selectionLabel ""
    set singularValues 1
    set plane x
    set interactive 1
    if {[APSStrictParseArguments {input singularValues plane deleteInput title selectionLabel interactive}] || \
          $singularValues<1 || [lsearch -exact [list x y] $plane]==-1 } {
        return -code error "APSSRFindOrbitGlitchSourceSVD: invalid options"
    }

    if $interactive {
        global apsSRFindOrbitGlitchSourceSVDSingularValues
        global apsSRFindOrbitGlitchSourceSVDPlane
        global apsSRFindOrbitGlitchSourceSVDDialogVar
        set apsSRFindOrbitGlitchSourceSVDSingularValues $singularValues
        set apsSRFindOrbitGlitchSourceSVDPlane $plane
        set apsSRFindOrbitGlitchSourceSVDDialogVar 0
        set w [APSUniqueName .]
        APSDialogBox $w -name "SVD Analysis Setup"  \
          -okCommand "set apsSRFindOrbitGlitchSourceSVDDialogVar 1" \
          -cancelCommand "set apsSRFindOrbitGlitchSourceSVDDialogVar 2"
        APSLabeledEntry .le1 -parent $w.userFrame \
          -label "Number of singular values: " \
          -type integer -textVariable apsSRFindOrbitGlitchSourceSVDSingularValues 
        APSRadioButtonFrame .rb1 -parent $w.userFrame \
          -label "Plane for analysis:        " -orientation horizontal \
          -variable apsSRFindOrbitGlitchSourceSVDPlane \
          -buttonList "x y" -valueList "x y"
        tkwait variable apsSRFindOrbitGlitchSourceSVDDialogVar
        update
        if $apsSRFindOrbitGlitchSourceSVDDialogVar==2 return
        set singularValues $apsSRFindOrbitGlitchSourceSVDSingularValues
        set plane $apsSRFindOrbitGlitchSourceSVDPlane
    }

    # Should choose the matrix based on the date of the file
    if [string compare $plane x]==0 {
        set modelMatrix /home/helios/oagData/sr/lattices/default/sr.h1rm
        set xy x
        set corrFilter S*H*,etax
    } else {
        set modelMatrix /home/helios/oagData/sr/lattices/default/sr.v1rm
        set xy y
        set corrFilter S*V*
    }

    set tmpRoot /tmp/[APSTmpString]
    set filter S*:P*:*$plane
    # DC data must be removed to prevent DC vector from dominating the reconstruction
    if [catch {exec sddsprocess $input -pipe=out -retain=col,${filter} \
                 -proc=$filter,average,%sAve \
                 "-redefine=col,%s,%s %sAve -,select=${filter}" \
                 | sddspseudoinverse -oldColumnNames=SignalName -digits=4 \
                 -pipe=in $tmpRoot.inv \
                 -largestSingularValues=$singularValues \
                 -sfile=$tmpRoot.s -vmatrix=$tmpRoot.v \
                 -reconstruct=$tmpRoot.recon} result] {
        return -code error "APSSRFindOrbitGlitchSourceSVD: $result"
    }

    # Change names in .v file to eliminate the PV name suffix from the signal names
    # and xref in the positions from the matrix file.
    # Also gets rid of any data for BPMs that are not in the model matrix.
    exec sddsprocess $tmpRoot.v -pipe=out -reedit=column,SignalName,2S/:/1000d \
      | sddsxref -pipe -nowarning $modelMatrix -match=SignalName=BPMName -take=s \
      | sddsprocess -pipe "-define=column,Sector,s 27.6 /" \
      | sddssort -pipe -column=s,increasing \
      | sddsconvert -pipe=in $tmpRoot.v1 -delete=column,s 

    # Create a normalized response matrix with only the BPMs that are present
    # in the data.
    exec sddsselect $modelMatrix $tmpRoot.v1 -pipe=out -match=BPMName=SignalName \
      | sddssort -pipe -column=s,increasing \
      | sddsnormalize -pipe -column=mode=rms,$corrFilter \
      | sddsconvert -pipe=in $tmpRoot.norm -retain=column,BPMName,$corrFilter
    
    for {set isv 0} {$isv<$singularValues} {incr isv} {
	lappend SVList SV[format %04ld $isv]
    }
    
    exec sddsplot $tmpRoot.s -column=Index,SingularValues -mode=y=log,y=spec \
      -graph=bar  -clip=19,0,invert \
      "-title=Singular values for $plane plane" \
      "-topline=$title  $selectionLabel" &
    exec sddsplot $tmpRoot.v1 -column=Sector,([join $SVList ,]) -separate=page -split=page -same \
      "-title=Singular vectors for $plane plane" \
      "-topline=$title  $selectionLabel" &
    
    for {set SV 0} {$SV<$singularValues} {incr SV} {
        set SVname SV[format %04ld $SV]

        # Make a file containing this SV vector only 
        exec sddsconvert $tmpRoot.v1 $tmpRoot.2 -retain=column,$SVname

        # Make a file containing the corrector names in a column
        exec sddsconvert $modelMatrix -retain=col,$corrFilter $tmpRoot.corrnames1
        exec sddsquery $tmpRoot.corrnames1 -pipe=out -column -sddsOutput \
          | sddsconvert -pipe=in $tmpRoot.corrnames \
          -retain=col,Name -rename=column,Name=Corrector
        file delete $tmpRoot.corrnames1

        # Take the product Transpose(M).SV, there "M" is the normalized response
        # matrix just formed.
        # Find the correctors for which this product is largest in absolute value.
        if [catch {exec sddstranspose $tmpRoot.norm -pipe=out \
                     | sddsconvert -pipe -delete=column,OldColumnNames \
                     | sddsmatrixmult $tmpRoot.2 -pipe \
                     | sddsprocess -pipe "-define=column,AbsProduct,$SVname abs" \
                     | sddsxref -pipe $tmpRoot.corrnames -take=* \
                     | sddssort -pipe -column=AbsProduct,decreasing \
                     | sddsprocess -pipe -clip=19,0,invert \
                     | sddsprintout \
                     "-title=$title  $selectionLabel\nPower supplies in order of decreasing overlap with glitch" \
                     -pipe=in -column $tmpRoot.print} result] {
            return -code error "APSSRFindOrbitGlitchSourceSVD: $result"
        }
        APSFileDisplayWindow [APSUniqueName .] -comment "SV $SV" \
          -fileName $tmpRoot.print -width 80 -height 20 -modal 1
    }
    if $deleteInput {
        file delete -force $input
    }
}


