#!/bin/sh  
# \
exec oagtclsh "$0" "$@"

set auto_path [linsert $auto_path 0  /usr/local/oag/apps/lib/$env(HOST_ARCH)]
set auto_path [linsert $auto_path 0 /usr/local/oag/lib_patch/$env(HOST_ARCH)]
APSStandardSetup


set usage "usage: csub \[-priority <number>\] \[-openmp <threads>\] \[-mpich3 <threads>\] \[-mpich-3.3.2 <threads>\] \[-mpich-3.3.1 <threads>\] \[-mpi <threads>\] \[-mvapich2 <threads>\] \[-mvapich2-1.7 <threads>\] \[-mvapich2-1.9 <threads>\]  \[-mvapich2-2.2 <threads>\] \[-mvapich2-2.3b <threads>\] \[-mvapich2-2.3.2 <threads>\] \[-lam <threads>\] \[-opal <threads>\] \[-name <string>\] \[-hostList <listOfNames>\]  \[-noEmail 1\] \[-stdin <filename\] \[-hive 1\] \[-apex 1\] \[-gpu 1\] \[-holdJob <jobID>\] \[-semaphore <filename>\] <command>\nSubmits a command to SGE."

set holdJob ""
set group ""
set priority 0
set args ""
set name ""
set index 0
set noEmail 0
set hostList ""
set lam 0
set mpi 0
set mpich2 0
set openmpi 0
set pvm 0
set totalview 0
set mpich2large 0
set mpich2_mpd_sock 0
set mpich2_smpd_nemesis 0
set mpich2_mpd_nemesis 0
set mvapich2 0
set mvapich2-1.7 0
set mvapich2-1.9 0
set mvapich2-2.2 0
set mvapich2-2.3b 0
set mvapich2-2.3.2 0
set stdin ""
set hive 0
set apex 0
set gpu 0
set semaphore ""
set mpich3 0
set mpich-3.3.1 0
set mpich-3.3.2 0
set openmp 0
set opal 0

while {$index<[llength $argv]} {
    if {[string match -* [lindex $argv $index]]} {
	lappend args [lindex $argv $index]
	incr index
	lappend args [lindex $argv $index]
	incr index
    } else {
	break
    }
}
if [llength $args] {
    APSStrictParseArguments {group priority holdJob name hostList noEmail lam mpi mpich2 openmpi mpich2large pvm totalview mpich2_mpd_sock mpich2_smpd_nemesis mpich2_mpd_nemesis mvapich2 {mvapich2-1.7} {mvapich2-1.9} {mvapich2-2.2} {mvapich2-2.3b} {mvapich2-2.3.2} stdin hive apex gpu mpich3 {mpich-3.3.1} {mpich-3.3.2} semaphore openmp opal}
    set argv [lrange $argv $index end]
}
if [llength $argv]==0 {
    puts stderr "$usage"
    exit 1
}


set command " "
set first 1
foreach item $argv {
    if {$first} {
        set first 0
        set item [exec which $item]
	if {[string range $item 0 6] == "/head2/"} {
            set item [string range $item 6 end]
        } elseif {[string range $item 0 5] == "/head/"} {
            set item [string range $item 5 end]
        }
    }
    if {$item == "<"} {
        append command $item
    } else {
        append command \\"$item\\"
    }
    append command " "
}

set name [string trim $name]
if [string length $name]==0 {
    set name [os editstring 10Z/ [lindex [lindex $argv 0] 0]]
}

if [string match "solaris*" $env(HOST_ARCH)] {
    set option1 "-cwd -j y"
} else {
    set option1 "-V -cwd -j y"
}

if {[string length $stdin] != 0} {
    set stdin " -stdin $stdin "
}

if $openmp!=0 {
    append option1 " -pe openmp $openmp"
}
if $lam!=0 {
    append option1 " -pe lam $lam"
    set command "/share/lam-7.1.2/bin/mpirun C $command"
}

if $mpi!=0 {
    append option1 " -pe mpi $mpi"
    set command "/act/mvapich-1.2rc1-gcc-g95/bin/mpirun -np $mpi -machinefile \\\$TMPDIR/machines $stdin $command"
    set apex 1
}
if $mpich3!=0 {
    append option1 " -pe mpich3 $mpich3"
    set command "/lustre/3rdPartySoftware/mpich-3.0.2/bin/mpirun -np $mpich3 -machinefile \\\$TMPDIR/machines $stdin $command"
}
if {${mpich-3.3.1} != 0} {
    append option1 " -pe mpich3 ${mpich-3.3.1}"
    set command "/lustre/3rdPartySoftware/mpich-3.3.1/bin/mpirun -np ${mpich-3.3.1} -machinefile \\\$TMPDIR/machines $stdin $command"
}
if {${mpich-3.3.2} != 0} {
    append option1 " -pe mpich3 ${mpich-3.3.2}"
    set command "/lustre/3rdPartySoftware/mpich-3.3.2/bin/mpirun -np ${mpich-3.3.2} -machinefile \\\$TMPDIR/machines $stdin $command"
}
if {$mvapich2 != 0} {
    append option1 " -pe mvapich2_gnu $mvapich2"
    set command "/act/mvapich2/gnu/bin/mpirun_rsh -rsh -hostfile \\\$TMPDIR/machines -np $mvapich2 MV2_ENABLE_AFFINITY=0 MV2_ON_DEMAND_THRESHOLD=5000 $command"
}
if {${mvapich2-1.7} != 0} {
    append option1 " -pe mvapich2_gnu ${mvapich2-1.7}"
    set command "/act/mvapich2-1.7/bin/mpiexec -machinefile \\\$TMPDIR/machines -np ${mvapich2-1.7} $command"
}
if {${mvapich2-1.9} != 0} {
    append option1 " -pe mvapich2_gnu ${mvapich2-1.9}"
    if {$hive} {
        append option1 " -hard -l excl=true"
    }

    if {[string length $stdin] != 0} {
        append command " < [lindex $stdin 1]"
    }
    set command "/lustre/3rdPartySoftware/mvapich2-1.9rc1/bin/mpirun_rsh -rsh -hostfile \\\$TMPDIR/machines -np ${mvapich2-1.9} MV2_ENABLE_AFFINITY=0 MV2_ON_DEMAND_THRESHOLD=5000 $command"
}
if {${mvapich2-2.2} != 0} {
    append option1 " -pe mvapich2_gnu ${mvapich2-2.2}"
    if {$hive} {
        append option1 " -hard -l excl=true"
    }

    if {[string length $stdin] != 0} {
        append command " < [lindex $stdin 1]"
    }
    set command "/lustre/3rdPartySoftware/mvapich2-2.2/bin/mpirun_rsh -rsh -hostfile \\\$TMPDIR/machines -np ${mvapich2-2.2} MV2_ENABLE_AFFINITY=0 MV2_ON_DEMAND_THRESHOLD=5000 $command"
}
if {${mvapich2-2.3b} != 0} {
    append option1 " -pe mvapich2_gnu ${mvapich2-2.3b}"
    if {$hive} {
        append option1 " -hard -l excl=true"
    }

    if {[string length $stdin] != 0} {
        append command " < [lindex $stdin 1]"
    }
    set command "/lustre/3rdPartySoftware/mvapich2-2.3b/bin/mpirun_rsh -rsh -hostfile \\\$TMPDIR/machines -np ${mvapich2-2.3b} MV2_ENABLE_AFFINITY=0 $command"
}
if {${mvapich2-2.3.2} != 0} {
    append option1 " -pe mvapich2_gnu ${mvapich2-2.3.2}"
    if {$hive} {
        append option1 " -hard -l excl=true"
    }

    if {[string length $stdin] != 0} {
        append command " < [lindex $stdin 1]"
    }
    set command "/lustre/3rdPartySoftware/mvapich2-2.3.2/bin/mpirun_rsh -rsh -hostfile \\\$TMPDIR/machines -np ${mvapich2-2.3.2} MV2_ENABLE_AFFINITY=0 $command"
}

if $pvm!=0 {
    append option1 " -pe pvm $pvm"
}
if $openmpi!=0 {
    append option1 " -pe openmpi $openmpi"
    puts "openmpi not installed on cluster"
    exit
}
if $opal!=0 {
    append option1 " -pe opal $opal"
    set command "/lustre/3rdPartySoftware/OPAL-2.2.0/bin/mpirun -mca plm_rsh_force_rsh 1 -hostfile \\\$TMPDIR/machines -np $opal $command"
}

if {[string length $holdJob] > 0} {
    append option1 " -hold_jid $holdJob"
}


if [string match "linux*" $env(HOST_ARCH)] {
    if {$group == "a"} {
	set hostList [list all.q@weed43 all.q@weed44 all.q@weed45 all.q@weed46 all.q@weed47 all.q@weed48 all.q@weed49 all.q@weed50 all.q@weed65 all.q@weed66 all.q@weed67 all.q@weed68 all.q@weed69 all.q@weed70 all.q@weed71 all.q@weed72 all.q@weed73 all.q@weed74 all.q@weed75 all.q@weed76 all.q@weed77 all.q@weed78 all.q@weed79 all.q@weed80 all.q@weed81 all.q@weed82 all.q@weed83 all.q@weed84 ]
    } elseif {$group == "b"} {
	set hostList [list all.q@weed25 all.q@weed26 all.q@weed27 all.q@weed28 all.q@weed29 all.q@weed30 all.q@weed31 all.q@weed32 all.q@weed33 all.q@weed34 all.q@weed35 all.q@weed36 all.q@weed37 all.q@weed38 all.q@weed39 all.q@weed40 all.q@weed41 all.q@weed42 ]
    } elseif {$group == "c"} {
	set hostList [list all.q@weed1 all.q@weed2 all.q@weed3 all.q@weed4 all.q@weed5 all.q@weed6 all.q@weed7 all.q@weed8 all.q@weed9 all.q@weed10 all.q@weed11 all.q@weed12 all.q@weed13 all.q@weed14 all.q@weed15 all.q@weed16 all.q@weed18 all.q@weed19 all.q@weed20 all.q@weed21 all.q@weed22 all.q@weed23 all.q@weed24 ]
    }
}




if [string length $hostList] {
    set optionQ "-hard -q [join $hostList ,]"
} elseif {$apex} {
    set optionQ "-hard -q apex.q"
} elseif {$hive} {
    set optionQ "-hard -q hive.q"
} elseif {$gpu} {
    set optionQ "-hard -q gpu.q"  
} else {
    set optionQ "-hard -q all.q"
}

if [string length $semaphore] {
    if [file exists $semaphore] {
        return -code error "semaphore file $semaphore already exists"
    }
    set semaphoreCommand "echo done > $semaphore\n"
} else {
    set semaphoreCommand "\n"
}
#puts "$command"
#puts "qsub $option1 -p $priority -N $name $optionQ"
#exit

if $noEmail {
    if [catch {eval exec echo \"uname -a\n$command\necho $command done in [pwd]\n$semaphoreCommand\" \
		   | qsub $option1 -p $priority -N $name $optionQ} result] {
	puts stderr $result
	exit 1
    }
} else {
    if [catch {eval exec echo \"uname -a\n$command\n${semaphoreCommand}echo $command done in [pwd] | mail -s \\"$command done\\" $env(USER)\nsleep 2\" \
		   | qsub $option1 -p $priority -N $name $optionQ} result] {
	puts stderr $result
	exit 1
    }
}
puts stdout "${command} ([lindex $result 2]) submitted."
