Hack to automatically restart programs that stall during inicialization

    #!/bin/bash -l
    #PBS -l mem=100gb,nodes=100:ppn=1,walltime=03:59:59
    #PBS -m a
    #PBS -j oe

    # This method checks to see if a file is created at the beginning of a run. If the file is not created then the job is killed and restarted.
    # This method assumes that the program stalls before it has a chance to generate a file. This method is useful if the code does self checkpointing and the failure state is a busy wait.
    cd ${PBS_O_WORKDIR}
     
    #Run the main program
    ( 
        mpirun mycommand 
   ) &
    PID=$!

    # Sleep for enough time to see if the job is running
    sleep 300
    if [ ! -f testfile.flag ]
    then
             echo "Job Seems to have stalled. Killing and restarting"
             kill $PID
             qsub $0
             echo "Job stats for debugging"
             qstat -f ${PBS_JOBID}
             exit 1
    fi

    wait $PID
    RET=$?
    qstat -f ${PBS_JOBID}


    #return the output of the main program
    exit $RET

    #!/bin/bash -l
    #PBS -l mem=100gb,nodes=100:ppn=1,walltime=03:59:59
    #PBS -m a
    #PBS -j oe
    # This method monitors job output and stops if the output doesn't change. 
    # This method assumes that the program continuously generates output at regular intervals. 
    cd ${PBS_O_WORKDIR}

    testfile=`testfile.flag`
     
    #Run the main program
    ( 
        mpirun mycommand > $testfile
   ) &
    PID=$!

    # Sleep for enough time to start generating output
    sleep 300
    linecount1=`cat $testfile | wc -l`
     
    # Sleep enough for more output
    sleep 100
    linecount2=`cat $testfile | wc -l`
    if [ "$linecount1" == "$linecount2" ]
    then
             echo "Job Seems to have stalled. Killing and restarting"
             kill $PID
             qsub $0
             echo "Job stats for debugging"
             qstat -f ${PBS_JOBID}
             exit 1
    fi

    wait $PID
    RET=$?
    qstat -f ${PBS_JOBID}


    #return the output of the main program
    exit $RET

    #!/bin/bash -l
    #PBS -l mem=100gb,nodes=100:ppn=1,walltime=03:59:59
    #PBS -m a
    #PBS -j oe
    # This method uses the same idea as the previous but instead of relying on output it uses the cput stat generated by qstat.
    # This solution will not work if the job is in a busy wait state. 
    cd ${PBS_O_WORKDIR}

    #Run the main program
    ( 
        mpirun mycommand 
   ) &
    PID=$!

    # Sleep for enough time to start generating output
    sleep 300
    cpu1=`qstat -f $PBS_JOBID | grep resources_used.cput`
     
    # Sleep enough for more output
    sleep 100
    cpu2=`qstat -f $PBS_JOBID | grep resources_used.cput`
    if [ "$cpu1" == "$cpu2" ]
    then
             echo "Job Seems to have stalled. Killing and restarting"
             kill $PID
             qsub $0
             echo "Job stats for debugging"
             qstat -f ${PBS_JOBID}
             exit 1
    fi

    wait $PID
    RET=$?
    qstat -f ${PBS_JOBID}


    #return the output of the main program
    exit $RET