#!/bin/sh  
# \
  exec oagtclsh "$0" "$@"


if ![info exists env(OAG_TOP_DIR)] {
    set env(OAG_TOP_DIR) /usr/local
}
set auto_path [linsert $auto_path 0  $env(OAG_TOP_DIR)/oag/apps/lib/$env(HOST_ARCH)]

APSStandardSetup

set usage {usage: genOpt -input <filename> [-contin 1] [-errorFactor <number>] [-annealingRate <rate>] [-reduce 1] [-help 1] [-multiObjective <1|0>] [-topUp 1] [-maxRank <integer>] [-drain 1] [-updateOnly 1] [-pulse <number>] [-workstationMode 1] [-autoPulse <script>]}

proc PrintHelp {} {
    global usage
    puts stdout "$usage\n"
    puts stdout "Description of commandline parameters:"
    puts stdout "input:  name of the input file, as described above."
    puts stdout "contin: if non-zero, then continue optimizing from a previous run"
    puts stdout "errorFactor: factor by which to multiply the errorLevels from the input"
    puts stdout "  file.  "
    puts stdout "annealingRate: reduces the error factor according to the given rate, 0<=R<1."
    puts stdout "  If the initial factor is F, then after N steps the value will be F*(1-R)^N."
    puts stdout "reduce: if non-zero, then trial jobs that are not selected for breeding"
    puts stdout "  are deleted.  This saves time and disk space."
    puts stdout "multiObjective: if non-zeron, then perform multiobjective optimization."
    puts stdout "   The post-processing file *.proc should contain penalty value columns"
    puts stdout "   one for each object, ending with the string \"Penalty\".  It may or"
    puts stdout "   may not contain ConstraintViolation (short) columns."
    puts stdout "topUp: if non-zero, then for continued runs the optimizer will start a sufficient number of jobs to ensure that"
    puts stdout "   the desired total number is running.  Use this only if you have changed you input file in the middle of"
    puts stdout "   running to increase the number of jobs and want to add jobs now, rather than waiting for processing to occur."
    puts stdout "maxRank: for multiobjective mode, maximum rank of solutions to include in breeding. Default is 1."
    puts stdout "drain: script waits for jobs to complete and updates output files, but doesn't submit new jobs."
    puts stdout "updateOnly: script updates output files and then exits."
    puts stdout "pulse: add a one-time pulse of <number> jobs"
    puts stdout "workstationMode: run the jobs locally."
    puts stdout "autoPulse: use script to determine whether to emit a pulse of jobs"
    puts stdout ""
    puts stdout "Description of input file:"
    puts stdout "Parameters: "
    puts stdout "nTotalJobs:  maximum number of jobs to run"
    puts stdout "nParents:    number of parent jobs to use for breeding."
    puts stdout "childMultiplier: number of child jobs to create per parent"
    puts stdout "    NB: childMultiplier*nParents gives the number of simultaneous"
    puts stdout "    jobs that will be run on the queue."
    puts stdout "    For multi-objective mode, only this product is important."
    puts stdout "maxRank: overrides maxRank parameter on the commandline."
    puts stdout "sleepTime:   seconds to sleep between checking for newly-completed jobs"
    puts stdout "staggerTime:  seconds to sleep between submission of successive jobs,"
    puts stdout "    which can help reduce peak load on the system."
    puts stdout "multiObjective: if non-zero, perform multiobjective optmization.  See above for"
    puts stdout "    requirements.  Over-rides any commandline value."
    puts stdout "preProcessingScript: script to run prior to running the job. (May be blank.)"
    puts stdout "  The script must accept the following arguments and syntax: "
    puts stdout "    -rootname <string>       rootname for the job."
    puts stdout "    -tagList <string>        list of tags for varied quantities."
    puts stdout "    -valueList <string>      list of values associated with tags."
    puts stdout "    The tag names are the same as the parameter names given in the"
    puts stdout "    parameterName column (see below)."
    puts stdout "  E.g., the script will be called with arguments like these:"
    puts stdout "    -rootname run-000000 -tagList \"Var1 Var2\" -valueList \"1.7 -2.8\""
    puts stdout "postProcessingScript: script to run to postprocess a job.  Called with"
    puts stdout "  one argument giving the rootname of the job.  This script must produce"
    puts stdout "  a one-row SDDS file named <rootname>.proc that contains at least the "
    puts stdout "  following columns:"
    puts stdout "  runName --- the rootname of the run for this file"
    puts stdout "  penaltyValue --- the value of the penalty function"
    puts stdout "  This file must also have all the columns from <rootname>.inp sddsxref'd in."
    puts stdout "runScript: script to run the simulation job.  The script must accept the"
    puts stdout "  same arguments as the preProcessingScript.  If this is blank, then you"
    puts stdout "  must give inputFileTemplate, inputFileExtension, and programName."
    puts stdout "  The script must create a file <rootname>.run as soon as it starts"
    puts stdout "  and <rootname>.done just before finishing."
    puts stdout "inputFileTemplate: name of the template file for creating input files."
    puts stdout "  Sequences of the form <name> are substituted with values."
    puts stdout "inputFileExtension: extension to use for input files."
    puts stdout "programName: name of the program to run with the input files.  The"
    puts stdout "  input filename is given as the sole argument of the program."
    puts stdout "progressScript: name of a script to run when new rank=1 solutions are found"
    puts stderr "autoPulse: name of a script to use to determine how many jobs to emit in a pulse"
    puts stdout ""
    puts stdout "Columns:"
    puts stdout "parameterName: names of the parameters to vary in the optimization."
    puts stdout "  These should appear in the input file template in the form <name>."
    puts stdout "initialValue: initial values of the parameters"
    puts stdout "errorLevel: rms width of the gaussian random errors to add to the"
    puts stdout "  parameter values as mutations."
    puts stdout "lowerLimit: smallest allowable values for the parameters"
    puts stdout "upperLimit: largest allowable values for the parameters."
}

set args $argv
set contin 0
set input ""
set errorFactor 1
set annealingRate 0
set reduce 0
set help 0
set breedingMethod 1
set multiObjective 0
set updateOnly 0
set topUp 0
set drain 0
set maxRank 1
set pulse 0
set autoPulse ""
set workstationMode 0
if {[APSStrictParseArguments {contin input errorFactor annealingRate reduce help breedingMethod multiObjective topUp updateOnly drain maxRank pulse autoPulse workstationMode}] || \
      ![string length  $input] || $errorFactor<=0 || [expr $annealingRate>=1] || [expr $annealingRate<0]} {
    if $help {
        PrintHelp
        exit 
    }    
    return -code error "$usage"
}


if ![file exists $input] {
    return -code error "not found: $input"
}
set rootname [file rootname $input]

set preexistingJobs [llength [glob -nocomplain $rootname-??????.*]]
if {!$contin && $preexistingJobs} {
    return -code error "rootname in use: $rootname"
}

proc SubmitJob {args} {
    set code ""
    set input "" 
    APSStrictParseArguments {code input}

    if {[catch {exec /usr/bin/which ssub} results]} {
        puts stdout "ssub not found"
        exit
    }

    catch {exec ssub -partition hive -name [file root [file tail $input]] $code $input} result

    return "$result"

    if {0} {
        if {[catch {exec /usr/bin/which qsub} results]} {
            puts stdout "qsub not found"
            exit
        }
        
        eval file delete -force $input.log [file rootname $input].done 
        set tmpFile [file rootname $input].csh
        set fd [open $tmpFile w]
        puts $fd "#!/bin/csh "
        puts $fd "source ~/.cshrc"
        puts $fd "unset savehist"
        puts $fd "echo running $code $input on [exec uname -a]"
        puts $fd "cd [pwd]"
        puts $fd "$code $input >& $input.log"
        close $fd
        
        catch {exec cat $tmpFile | qsub -V -o [pwd] -j y -N [file root [file tail $input]] -hard -q all.q } result
        
        return "$result"
    }
}

proc SubmitRunScript {args} {
    global workstationMode
    set script ""
    set tagList ""
    set valueList ""
    set rootname ""
    APSStrictParseArguments {script rootname tagList valueList}

    eval file delete -force $rootname.log $rootname.done $rootname.run
    if {$workstationMode} {
        set tmpFile [file rootname $rootname].csh
        APSAddToTempFileList $tmpFile
        set fd [open $tmpFile w]
        puts $fd "#!/bin/csh "
        puts $fd "echo running $script $rootname $tagList $valueList on [exec uname -a]"
        puts $fd "cd [pwd]"
        puts $fd "$script -rootname $rootname -tagList '$tagList' -valueList '$valueList' >& $rootname.log"
        close $fd
        catch {exec chmod u+x $tmpFile } result
        catch {exec $tmpFile & } result
    } else {
        if {[catch {exec /usr/bin/which ssub} results]} {
            puts stdout "ssub not found"
            exit
        }
        catch {exec ssub -partition hive -name [file root [file tail $rootname]] $script -rootname $rootname -tagList \\\"$tagList\\\" -valueList \\\"$valueList\\\"} result

        if {0} {
            if {[catch {exec /usr/bin/which qsub} results]} {
                puts stdout "qsub not found"
                exit
            }
            set tmpFile [file rootname $rootname].csh
            APSAddToTempFileList $tmpFile
            set fd [open $tmpFile w]
            puts $fd "#!/bin/csh "
            puts $fd "source ~/.cshrc"
            puts $fd "unset savehist"
            puts $fd "echo running $script $rootname $tagList $valueList on [exec uname -a]"
            puts $fd "cd [pwd]"
            puts $fd "$script -rootname $rootname -tagList '$tagList' -valueList '$valueList' >& $rootname.log"
            close $fd
            
            catch {exec cat $tmpFile | qsub -V -o [pwd] -j y -N [file root [file tail $rootname]] -hard -q all.q } result
        }
    }
    return "$result"
}

proc LoadAndCheckInputFile {} {
    global input inputParameterList inputColumnList
    global staggerTime
    global executionMode generateRunScript permissionToSubmitScript permissionToSubmitTries 
    set staggerTime 0
    set generateRunScript ""
    set permissionToSubmitScript ""
    set permissionToSubmitTries -1
    if [catch {sdds load $input inputData} result] {
        return -code error "$result"
    }
    set inputParameterList $inputData(ParameterNames)
    foreach param $inputParameterList {
        global $param
        set $param [lindex $inputData(Parameter.$param) 0]
    }
    set inputColumnList $inputData(ColumnNames)
    foreach col $inputColumnList {
        global $col
        set $col [lindex $inputData(Column.$col) 0]
    }
    foreach newParam {runScript preProcessingScript} {
        if ![info exists $newParam] {
            global $newParam
            lappend inputParameterList $newParam
            set $newParam ""
        }
    }
    set count 0
    foreach item {runScript programName generateRunScript} {
	if [string length [set $item]] {
	    set executionMode $item
	    set count 1
	}
    }
    if $count!=1 {
	puts stderr "Error: give one and only one of runScript programName generateRunScript"
	exit 1
    }
    if [string length $runScript] {
        set inputFileExtension run
    }
    
    foreach pN $parameterName iV $initialValue lL $lowerLimit uL $upperLimit {
        if {$lL>$uL} {
            puts stderr "Error: $pN lower limit ($lL) is greater than the upper limit ($uL)."
            exit 1
        }
        if {$iV>$uL || $iV<$lL} {
            puts stderr "Error: $pN initial value ($iV) is out of range ($lL , $uL)."
            exit 1
        }
    }
}

proc grndl {limit} {
    set pi 3.14159265
    while 1 {
        set u1 [expr rand()]
        set u2 [expr rand()]
        set value [expr cos(2*$pi*$u2)*sqrt(-2*log($u1))]
        if [expr abs($value)<=$limit] {
            return $value
        }
    }
}

proc StartInitialJobs {args} {
    set offset 0
    set startID 0
    APSStrictParseArguments {offset startID}
    global inputParameterList inputColumnList 
    eval global $inputParameterList 
    eval global $inputColumnList
    global rootname template errorFactor jobsStarted staggerTime executionMode generateRunScript pulse autoPulse

    set jobID [expr $startID-1]
    for {set job $offset} {$pulse>0 || $job<[expr $nParents*$childMultiplier]} {incr job} {
	incr jobID
        while 1 {
	    set runName [format $rootname-%06ld $jobID]
	    if [llength [glob -nocomplain ${runName}.*]]==0 {
		break
	    }
	    incr jobID
	}
	if ![WaitForPermissionToSubmit] return;
        set origOpt <rootname>
        set replOpt $runName
        set valueList ""
        set procOpt ""
        puts $job
        foreach pN $parameterName iV $initialValue eL $errorLevel lL $lowerLimit uL $upperLimit {
            lappend origOpt <$pN>
            if $job==0 {
                set value $iV
            } else {
                while 1 {
                    set value [expr [grndl 3]*$eL*$errorFactor+$iV]
                    if [expr $uL==$lL] break
                    if {[expr $value<=$uL] && [expr $value>=$lL]} break
                }
            }
            lappend replOpt $value
            lappend valueList $value
            lappend procOpt -define=column,$pN,$value,type=double
        }
        scan [os editstring e6b6kay100d $runName] %d runID
        catch {eval exec sddssequence -pipe -define=dummy,type=short -sequence=begin=0,end=0,number=1 \
                 | sddsprocess -pipe=in $runName.inp \
                 $procOpt \
                 -print=column,runName,$runName \
                 -define=column,runID,$runID,type=long } result
	if [string compare [exec sddscheck $runName.inp] "ok"]!=0 {
	    return -code error "$result"
	}
        if [string length $preProcessingScript] {
            if [catch {exec $preProcessingScript \
                         -rootname $runName \
                         -tagList "rootname $parameterName" \
                         -valueList "$replOpt"} result] {
                return -code error "$result"
            }
            if [string length $result] {
                puts stderr "$result"
            }
        }
        
        switch $executionMode {
            runScript {
                puts stderr "Submitting script: $runScript for $runName"
                catch {SubmitRunScript -script $runScript -rootname $runName \
                         -tagList "$parameterName" -valueList "$valueList"} result
            }
            programName {
                puts stderr "Submitting job"
                exec replaceText -original=[join $origOpt ,] -replacement=[join $replOpt ,] \
                  $inputFileTemplate $runName.$inputFileExtension
                catch {SubmitJob -code $programName -input $runName.$inputFileExtension \
                     } result
            }
	    generateRunScript {
		puts stderr "Generating and submitting script"
		catch {eval exec $generateRunScript -rootname $runName -tagList \"$parameterName\" \
                         -valueList \"$valueList\" -runID $runID} result
		puts stderr "$generateRunScript returns: $result"
	    }
	    default {
	    }
        }
        incr jobsStarted
	if [expr $pulse>0] {
	    incr pulse -1
	}
        puts stderr "Script submission returns: $result"
	if $staggerTime {
	    after [expr int($staggerTime*1000)]
	}
    }
}

proc StartChildJobs {} {
    global inputParameterList inputColumnList workstationMode
    eval global $inputParameterList 
    eval global $inputColumnList
    global rootname template jobsToStart jobsStarted jobsRunning staggerTime executionMode generateRunScript pulse autoPulse
    
    UpdateJobsRunning
    set jobsToStart [expr $nParents*$childMultiplier-$jobsRunning]
    set rows [exec sdds2stream -rows=bar $rootname.children]
    
    if [catch {sdds load $rootname.children childData} result] {
        return -code error "$result"
    }
    if {$jobsToStart>$rows} {
        set jobsToStart $rows
    }
    puts stderr "Try to start $jobsToStart jobs"
    for {set job 1} {$pulse>0 || $job<=$jobsToStart} {incr job} {
	if ![WaitForPermissionToSubmit] return
        set runName [format $rootname-%06ld $jobsStarted]
        set origOpt <rootname>
        set replOpt $runName
        set valueList ""
        set procOpt ""
        puts stderr "job $job"
        foreach pN $parameterName lL $lowerLimit uL $upperLimit {
            set value [lindex [lindex $childData(Column.$pN) 0] [expr $job-1]]
            if $lL!=$uL {
                if [expr $value>$uL] {
                    set value $uL
                } 
                if [expr $value<$lL] {
                    set value $lL
                }
            }
            lappend origOpt <$pN>
            lappend replOpt $value
            lappend valueList $value
            lappend procOpt -define=column,$pN,$value,type=double
            puts stderr "$pN -> $value"
        }
        scan [os editstring e6b6kay100d $runName] %d runID
        eval exec sddssequence -pipe -define=dummy,type=short -sequence=begin=0,end=0,number=1 \
          | sddsprocess -pipe=in $runName.inp \
          $procOpt \
          -print=column,runName,$runName \
          -define=column,runID,$runID,type=long 
	if [string compare [exec sddscheck $runName.inp] "ok"]!=0 {
	    return -code error "$result"
	}
        if [string length $preProcessingScript] {
            if [catch {exec $preProcessingScript \
                         -rootname $runName \
                         -tagList "$runName $parameterName" \
                         -valueList "$replOpt"} result] {
                return -code error "$result"
            }
            if [string length $result] {
                puts stderr "$result"
            }
        }

        switch $executionMode {
            runScript {
                puts stderr "Submitting script: $runScript for $runName"
                catch {SubmitRunScript -script $runScript -rootname $runName \
                         -tagList "$parameterName" -valueList "$valueList"} result
            }
            programName {
                puts stderr "Submitting job"
                exec replaceText -original=[join $origOpt ,] -replacement=[join $replOpt ,] \
                  $inputFileTemplate $runName.$inputFileExtension
                catch {SubmitJob -code $programName -input $runName.$inputFileExtension \
                     } result
            }
	    generateRunScript {
		puts stderr "Generating and submitting script"
		catch {eval exec $generateRunScript -rootname $runName -tagList \"$parameterName\" \
                         -valueList \"$valueList\" -runID $runID} result
		puts stderr "$generateRunScript returns: $result"
	    }
	    default {
	    }
        }
	if [expr $pulse>0] {
	    incr pulse -1
	}
        incr jobsStarted
        puts stderr "$result"
	if $staggerTime {
	    after [expr int($staggerTime*1000)]
	}
    }
    after 300
}

proc SelectParents {} {
    global inputParameterList inputColumnList 
    eval global $inputParameterList 
    eval global $inputColumnList
    global rootname template errorFactor reduce multiObjective nParents breedingMethod
    
    set doneList [lsort [glob -nocomplain $rootname-??????.done]]
    puts stderr "[llength $doneList] completed jobs"
    set procList [lsort [glob -nocomplain $rootname-??????.proc]]
    puts stderr "[llength $procList] processed jobs"
    foreach file $doneList {
        set runName [file rootname $file]
        scan [os editstring e6b6kay100d $runName] %d runID
        if {[string length $postProcessingScript] && ![file exists $runName.proc]} {
            puts stderr "Processing $runName ($runID) ..."
            if {[catch {eval exec $postProcessingScript -rootname $runName \
                          -runID $runID } result]} {
                puts stderr "Problem $runName processing: $result"
                eval file delete -force [glob $runName.*]
                continue
            }
	    if [file exist $runName.proc] {
                puts stderr "Processing completed"
		lappend procList $runName.proc 
	    } else {
		puts stderr "Anomalous processing failure for $runName"
	    }
        }
    }
    puts stderr "[llength $procList] processed jobs"
    if ![llength $procList] return
    
    puts stderr "Merging [llength $procList] processed jobs"
    if ![file exists $rootname.all] {
        eval exec sddscombine $procList -merge -pipe=out \
          | sddssort -pipe=in -col=runID -unique $rootname.all
    } else {
        eval exec sddscombine $rootname.all $procList -merge -pipe=out \
          | sddssort -pipe=in -column=runID -unique $rootname.all1
        file rename -force $rootname.all1 $rootname.all
    }
    if !$multiObjective {
        eval exec sddscombine $rootname.all -merge -pipe=out \
          | sddsprocess -pipe \
          "-print=col,pvString,%e,penaltyValue" \
          -match=col,pvString=+nan,! \
          -match=col,pvString=+inf,! \
          | sddssort -pipe -col=penaltyValue,incr \
          | sddsprocess -pipe=in $rootname.best -clip=$nParents,0,invert 
    } else {
	global maxRank
        set columns [exec sddsquery -col $rootname.all]
        set opt ""
        foreach col $columns {
            if [string match "*Penalty*" $col] {
                lappend opt -col=$col
            }
        }
	if $breedingMethod==0 {
	    # >2 parent breeding, using N top-ranked solutions
	    if [catch {eval exec sddssort -nondominate \
                         $opt $rootname.all $rootname.sort 
		exec sddsprocess $rootname.sort -filter=col,Rank,0,$maxRank $rootname.best1 } result] {
		puts stderr "$result"
		exit 1
	    }
	    set rows [exec sdds2stream -rows=bar $rootname.best1]
	    if {$rows<$nParents} {
		if [catch {exec sddsprocess $rootname.sort -clip=$nParents,0,invert $rootname.best} result] {
		    puts stderr "$result"
		    exit 1
		}
	    } else {
		file rename -force $rootname.best1 $rootname.best
	    }
	} else {
	    # 2-parent breeding, choosing from all top-ranked solutions
	    if [catch {eval exec sddssort -nondominate \
                         $opt $rootname.all $rootname.sort 
		exec sddsprocess $rootname.sort -define=param,maxRank,$maxRank -filter=col,Rank,0,$maxRank $rootname.best } result] {
		puts stderr "$result"
		exit 1
	    }
	}
    }
}

# In this algorithm a child may have more than two parents.
proc BreedNewJobs0 {} {
    global inputParameterList inputColumnList 
    eval global $inputParameterList 
    eval global $inputColumnList
    global rootname template errorFactor nParents
    
    set paramFileList ""
    set randomizeOptList ""
    foreach pN $parameterName eL $errorLevel {
        puts stderr "Preparing stock for breeding for $pN"
        eval exec sddscombine -merge \
          [APSReplicateItem -item $rootname.best -number $childMultiplier] \
          -pipe=out -retain=col,$pN \
          | sddsprocess -pipe -define=col,RN,rnd \
          | sddssort -pipe -col=RN \
          | sddsprocess -pipe=in $rootname.best.$pN -clip=$nParents,0,invert
        lappend paramFileList $rootname.best.$pN
        lappend randomizeOptList "-redefine=col,$pN,$pN 3 grndl $eL * $errorFactor * +"
    }
    puts stderr "Breeding"
    eval exec sddsxref $paramFileList -pipe=out \
      | sddsprocess -pipe=in $rootname.children $randomizeOptList
}

# This uses an algorithm whereby only two parents are bred 

proc BreedNewJobs1 {} {
    global inputParameterList inputColumnList 
    eval global $inputParameterList 
    eval global $inputColumnList
    global rootname template errorFactor

    foreach pN $parameterName eL $errorLevel {
        lappend procOptList -process=$pN,spread,%sSpread
        lappend procOptList -process=$pN,min,%sMin 
        lappend procOptList "-redefine=parameter,$pN,${pN}Spread rnd * ${pN}Min + 3 grndl $eL * $errorFactor * + "
    }

    set number1 [expr $childMultiplier*$nParents]
    set number [expr [llength $parameterName]*$childMultiplier]
    if $number>$number1 {
	set number $number1
    }
    eval exec sddscombine [APSReplicateItem -item $rootname.best -number $number] \
      -retain=column,[join $parameterName ,] -pipe=out \
      | sddsprocess -pipe -define=column,RN,rnd \
      | sddssort -pipe -column=RN \
      | sddsprocess -pipe -clip=2,0,invert $procOptList \
      | sddscollapse -pipe=in $rootname.children
}

proc RemoveLosers {} {
    global rootname template doneList jobsToStart errorFactor
    set bestRunList [APSGetSDDSColumn -column runName -fileName $rootname.best]
    foreach done $doneList {
        if [lsearch -exact $bestRunList [file rootname $done]]==-1 {
            puts "Removing [file rootname $done]"
            catch {eval file delete -force [glob [file rootname $done]]}
            catch {eval file delete -force [glob [file rootname $done].*]}
            catch {eval file delete -force [glob [file rootname $done]-*.*]}
        }
    }
}

proc UpdateJobsRunning {} {
    global rootname jobsRunning jobsStarted jobsToProc inputFileExtension jobsProc jobsCurrent pulse
    set jobsCurrent [llength [glob -nocomplain $rootname-??????.inp]]
    set jobsDone [llength [glob -nocomplain $rootname-??????.done]]
    set jobsProc [llength [glob -nocomplain $rootname-??????.proc]]
    set jobsToProc [expr $jobsDone-$jobsProc]
    set jobsRunning [expr $jobsCurrent-$jobsDone]
    set message "[clock format [clock seconds]]: Jobs: current=$jobsCurrent, done=$jobsDone, proc'd=$jobsProc, toProc=$jobsToProc, running=$jobsRunning"
    puts -nonewline stderr $message
    for {set i 0} {$i<[string length $message]} {incr i} {
	puts -nonewline stderr "\b"
    }
}

if [file exists geneticOptimizer.local] {
    source geneticOptimizer.local
}

proc WaitForPermissionToSubmit {} {
    global permissionToSubmitScript permissionToSubmitTries 
    if [string length $permissionToSubmitScript] { 
	set triesLeft $permissionToSubmitTries
	while {$triesLeft!=0} {
	    catch {exec $permissionToSubmitScript} result
	    switch $result {
		1 -
		yes {
		    return 1
		}
		default -
		0 - 
		no {
		    puts stderr "[clock format [clock seconds]]: Job submission not permitted now"
		    incr triesLeft -1
		    after 1000
		}
	    }
	}
	puts stderr "[clock format [clock seconds]]: Job submission wait timeout"
	return 0
    } else {
	return 1
    }
}

set progressScript ""
LoadAndCheckInputFile 
puts stderr "Input file read and checked."

if {!$contin} {
    if {[file exists $rootname.all]} {
        puts stderr "Error: $rootname.all file found but -contin 1 not given"
        exit 1
    }
    set jobsRunning 0
    set jobsStarted 0
    WaitForPermissionToSubmit
    StartInitialJobs
    after 3000
    set doWait 1
} else {
    set doWait 0
    if [catch {exec sddsprocess $rootname.all -pipe=out -process=runID,max,runIDMax \
                 | sdds2stream -pipe=in -parameter=runIDMax} result] {
        set result -1
    }
    set jobsStarted [expr int($result)+1]
    while {[llength [glob -nocomplain [format $rootname-%06d.* $jobsStarted]]]!=0} {
        incr jobsStarted
    }
    UpdateJobsRunning
    puts stderr "\n$jobsStarted jobs started for this optimization so far"
    puts stderr "$jobsToProc jobs need to be processed."
}

if $topUp {
    UpdateJobsRunning
    if $jobsRunning<[expr $childMultiplier*$nParents] {
	WaitForPermissionToSubmit
	StartInitialJobs -offset $jobsRunning -startID $jobsRunning
    }
}

set firstPass 1
set bestList ""
if [file exists $rootname.all] {
    set LastNRunSoFar [exec sdds2stream -rows=bare $rootname.all]
} else {
    set LastNRunSoFar 0
}

while {$jobsStarted<[expr $nTotalJobs+$nParents*$childMultiplier]} {
    if {$workstationMode} {
        after 300
    }
    UpdateJobsRunning
    if [string length $autoPulse] {
	set pulse [eval exec $autoPulse]
	if $pulse {
	    puts stderr "Preparing pulse of $pulse jobs"
	}
    }
    if {!$updateOnly && !($firstPass && $jobsRunning==0) && $jobsRunning>=[expr $childMultiplier*$nParents] && $pulse<=0} {
        continue
    }
    puts stderr "\n"
    set firstPass 0
    WaitForPermissionToSubmit
    puts stderr "Selecting parents..."
    set doneList [lsort [glob -nocomplain $rootname-??????.done]]
    if [catch {SelectParents} result] {
        puts stderr "$result"
    }
    set procList [lsort [glob -nocomplain $rootname-??????.proc]]
    if [file exists $rootname.best] {
        if $reduce {
            puts stderr "Removing losers..."
            if [catch {RemoveLosers} result] {
                puts stderr "$result"
                continue
            }
        }
	if $updateOnly exit
	if $drain {
	    after 3000
	    continue
	}
	set oldBestList $bestList
        if {!$multiObjective} {
            set bestList [exec sddssort $rootname.all -pipe=out -column=runID | sdds2stream -pipe -column=runID]
        } else {
            set bestList [exec sddsprocess $rootname.sort -pipe=out -filter=col,Rank,1,1 | sddssort -pipe -column=runID | sdds2stream -pipe -column=runID]
        }
	if {[llength $oldBestList]!=0 && [string compare [join $bestList ,] [join $oldBestList ,]]!=0} {
	    puts stderr "**[clock format [clock seconds]]:\n** Now [llength $bestList] rank-1 solutions: [join $bestList ,]\n**"
	    if [string length $progressScript] {
		if [catch {exec $progressScript -rootname $rootname -bestList [join $bestList ,]} result] {
		    puts stderr "$progressScript returns: $result"
		}
	    }
	}
        if [expr $annealingRate!=0] {
            set NRunSoFar [exec sdds2stream -rows=bare $rootname.all]
            set errorFactor [expr $errorFactor*pow(1.0-$annealingRate, $NRunSoFar-$LastNRunSoFar)]
            set LastNRunSoFar $NRunSoFar
            puts stderr "Error factor reduced to $errorFactor by annealing"
        }
        puts stderr "Breeding..."
        if [catch {BreedNewJobs$breedingMethod} result] {
            puts stderr "$result"
            continue
        }
        puts stderr "Starting new jobs..."
        if [catch {StartChildJobs} result] {
            puts stderr "$result"
            continue
        }
    }
}

