#!/bin/sh  
# \
exec oagtclsh "$0" "$@"

set auto_path [linsert $auto_path 0  /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]
APSStandardSetup

proc UsageMessage {} {
    puts stderr "Usage: ssub"
    puts stderr "         \[-mpi <mpich-4.1.1|mpich-4.0.3|mvapich2-2.3b|mvapich2-1.9>\]"
    puts stderr "         \[-name <jobname>\]"
    puts stderr "         \[-nthreads <number>\]"
    puts stderr "         \[-partition <haswell|haswellNonExclusive|broadwell|hive|hiveLowMem|apex|gpu>\]"
    puts stderr "         \[-chdir <directory>\]"
    puts stderr "         \[-exclusive 1\]"
    puts stderr "         \[-singleton 1\]"
    puts stderr "         \[-priority <value>\]"
    puts stderr "         \[-stdin <filename>\]"
    puts stderr "         <command>"
    puts stderr "Submits jobs to SLURM"
    puts stderr "Multiple partitons can also be requested: -partition haswell,broadwell,hive"
    puts stderr "The haswellNonExclusive is only intended for running large numbers for single"
    puts stderr "threaded jobs. The haswell partiton has exclusive mode turned on by default."
    puts stderr "Singleton limits the cluster to running only one job at a time with this unique name"

}
set mpi ""
set name ""
set nthreads 1
set partition "haswell"
set chdir ""
set exclusive 0
set priority ""
set stdin ""
set command ""
set mem-per-thread 0
set singleton 0

set index 0
set args ""
while {$index<[llength $argv]} {
    if {[string match -* [lindex $argv $index]]} {
	lappend args [lindex $argv $index]
	incr index
	lappend args [lindex $argv $index]
	incr index
    } else {
	break
    }
}
if [llength $args] {
    APSStrictParseArguments {mpi name nthreads partition chdir exclusive priority stdin mem-per-thread singleton}
    set argv [lrange $argv $index end]
}
if [llength $argv]==0 {
    UsageMessage
    exit 1
}

set command ""
set first 1
foreach item $argv {
    if {$first} {
        set first 0
        if {[catch {exec which $item} fullitem]} {
            puts stderr "$item is not in your path"
            exit 1
        }
        set item $fullitem
        set item [exec which $item]
	if {[string range $item 0 6] == "/head2/"} {
            set item [string range $item 6 end]
        } elseif {[string range $item 0 5] == "/head/"} {
            set item [string range $item 5 end]
        }
    }
    if {0} {
        if {$item == "<"} {
            append command $item
        } else {
            append command \\"$item\\"
        }
    } else {
        append command $item
    }
    append command " "
}

if {$mpi == "mpich-4.0.3"} {
    set command "/lustre/3rdPartySoftware/mpich-4.0.3/bin/mpirun $command"
}
if {$mpi == "mpich-4.1.1"} {
    set command "/lustre/3rdPartySoftware/mpich-4.1.1/bin/mpirun $command"
}
if {$mpi == "mvapich2-2.3b"} {
    set command "/lustre/3rdPartySoftware/mvapich2-2.3b/bin/mpirun $command"
    if {($partition == "haswell") || ($partition == "")} {
        set partition "hive"
    }
}
if {$mpi == "mvapich2-1.9"} {
    set command "/lustre/3rdPartySoftware/mvapich2-1.9rc1/bin/mpirun $command"
    if {($partition == "haswell") || ($partition == "")} {
        set partition "hive"
    }
}

set options ""
if {$name != ""} {
    append options "\"--job-name=$name\" "
}
if {$nthreads != 1} {
    append options "--ntasks=$nthreads "
}
if {${mem-per-thread} != 0} {
    append options "--mem-per-cpu=${mem-per-thread} "
}
if {$partition != ""} {
    append options "--partition=$partition "
}
if {$chdir != ""} {
    append options "--chdir=$chdir "
}
if {$exclusive == 1} {
    append options "--exclusive "
}
if {$singleton == 1} {
    append options "--dependency=singleton "
}
if {$priority != ""} {
    append options "--priority=$priority "
}
if {$stdin != ""} {
    append options "--input=$stdin "
}
set sbatchCommand "sbatch $options \"--wrap=$command\""
if {[catch {eval exec $sbatchCommand} results]} {
    puts stderr "Error: $results"
    exit 1
}
puts "$sbatchCommand"
puts "Job submitted to SLURM"

