Disclaimer: I do not use this script on TextDrive, due to resource consumption concerns, and would not recommend anyone else doing so without express permission from TPTB.
local threshold=5
local min_runtime=60
local init_sleep_time=5
local time_started time_ended
local sleep_time=${init_sleep_time}
local times_until_fail=${threshold}
until (( times_until_fail <= 0 )); do
time_started=$(date +%s)
$*
time_ended=$(date +%s)
if (( (${time_ended} - ${time_started}) > ${min_runtime} )); then
times_until_fail=${threshold}
sleep_time=${init_sleep_time}
fi
(( times_until_fail -= 1 ))
echo
echo "-- $(date +"%d/%m/%y %H:%M:%S"): \`$*\` died!"
if (( times_until_fail >= 1 )); then
echo "-- ${times_until_fail} more executions lasting less than ${min_runtime} seconds will result in job failure."
echo "-- Sleeping ${sleep_time} seconds..."
echo
sleep ${sleep_time}
fi
(( sleep_time *= 2 ))
done
echo "-- Died too many times in too short a space of time. Exiting!"