From 2a51e28983f10a4f7a2e56a7de9efcdec4eb6197 Mon Sep 17 00:00:00 2001 From: nfaguirrec Date: Mon, 5 Oct 2015 12:35:09 +0200 Subject: [PATCH 1/6] Corregido el bug en la linea command de HISTORY --- sjobq.del | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sjobq.del b/sjobq.del index a795123..e03ba1d 100755 --- a/sjobq.del +++ b/sjobq.del @@ -130,7 +130,8 @@ function showJobs() } ($1=="command"){ - command = sprintf("%s",$3) + gsub("command[[:blank:]]+= ", "") + command = sprintf("%s",$0) } END{ From e5804db2f3f30496f6bdcae040ac8616d795ff1a Mon Sep 17 00:00:00 2001 From: nfaguirrec Date: Mon, 5 Oct 2015 15:51:14 +0200 Subject: [PATCH 2/6] Fixed bug about stop and start functions --- sjobq.d | 25 ++++++++++++++++--------- sjobq.del | 13 ++++++++++--- sjobq.push | 6 +++--- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/sjobq.d b/sjobq.d index 8a121ad..150005f 100755 --- a/sjobq.d +++ b/sjobq.d @@ -153,8 +153,15 @@ function start() ## function stop() { + local id="" + + # @todo Maybe STOP_FILE it is not still necessary echo "" > $STOP_FILE - sleep 3 + sleep $REFRESH_INTERVAL + + # @todo It's neccessary to catch the output of kill command to avoid the message Killed + id="`ps -u $USER | grep "sjobq.d$" | awk '{print $1}'`" + kill -s SIGKILL $id &> /dev/null rm -rf $DATA_DIR } @@ -215,7 +222,7 @@ function main() then # It's already running. echo "### Error ### The daemon sjobq.d is already running" - echo " stop it using \"sjobq.d stop\"" + echo " You might want to run \"sjobq.d stop\" to stop it." exit 1 fi #------------------------------------ @@ -227,21 +234,21 @@ function main() nohup $0 __start > $DATA_DIR/log 2> $DATA_DIR/err & - echo "==============================" + echo "===============================" echo " SJobQ daemon has been started" - echo "==============================" + echo "===============================" ;; stop) isRunning="`ps -u $USER | grep "sjobq.d$" | awk '{a[NR]=$1}END{ if(a[2]==(a[1]+1)) print 0; else print 1}'`" if [ $isRunning -eq "0" ] then echo "### Error ### The daemon sjobq.d is not running" - echo " run it using \"sjobq.d start\"" + echo " You might want to run \"sjobq.d start\" to correct this." exit 1 else - echo "==============================" + echo "===============================" echo " SJobQ daemon has been stopped" - echo "==============================" + echo "===============================" stop fi @@ -262,9 +269,9 @@ function main() nohup $0 __start > $DATA_DIR/log 2> $DATA_DIR/err & - echo "================================" + echo "=================================" echo " SJobQ daemon has been restarted" - echo "================================" + echo "=================================" ;; __start) start diff --git a/sjobq.del b/sjobq.del index e03ba1d..b0b05bb 100755 --- a/sjobq.del +++ b/sjobq.del @@ -75,7 +75,7 @@ function showJobs() echo "" echo "pid = `cat $CURRENT_JOB_HOME/pid`" echo "command = `cat $CURRENT_JOB_HOME/com`" - echo "dir = `cat $CURRENT_JOB_HOME/pwd`" + echo "dir = `cat $CURRENT_JOB_HOME/pwd | sed 's/\/home\/'$USER'\//~\//g'`" echo "time spent = `diffTime $beginTime $endTime`" echo "pids = `cat $CURRENT_JOB_HOME/pids`" echo "tree =" @@ -198,7 +198,14 @@ function main() if [ -z "`ps -u $USER | grep "sjobq.d$"`" ] then echo "### Error ### The daemon sjobq.d is not running" - echo " run it using \"sjobq.d start\"" + echo " You might want to run \"sjobq.d start\" to correct this." + exit 1 + fi + + if [ ! -d "$DATA_DIR" ] + then + echo "### Error ### Data directory $DATA_DIR is empty" + echo " You might want to run \"sjobq.d restart\" to correct this." exit 1 fi @@ -213,4 +220,4 @@ function main() fi } -main $* \ No newline at end of file +main $* diff --git a/sjobq.push b/sjobq.push index 832ce7b..ab74b18 100755 --- a/sjobq.push +++ b/sjobq.push @@ -41,13 +41,13 @@ function main() if [ $previous -eq "32767" ] then - BID="0" + BID="1" else BID=$(( $previous + 1 )) fi else - echo "0" > $COUNTER_FILE + echo "1" > $COUNTER_FILE BID=`cat $COUNTER_FILE` fi echo $BID > $COUNTER_FILE @@ -56,7 +56,7 @@ function main() echo "id = $BID" echo "command = $COMMAND" - echo "dir = $PWD" + echo "dir = `echo $PWD | sed 's/\/home\/'$USER'\//~\//g'`" echo "$BID" > $DATA_DIR/$BID.bid echo "$COMMAND" > $DATA_DIR/$BID.com From c0328ca6aea48817e2f557ceb87f858773d77f97 Mon Sep 17 00:00:00 2001 From: nfaguirrec Date: Thu, 8 Oct 2015 17:42:52 +0200 Subject: [PATCH 3/6] Se ha mejorado el formato de salida --- sjobq.del | 126 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 107 insertions(+), 19 deletions(-) diff --git a/sjobq.del b/sjobq.del index d8db557..802f222 100755 --- a/sjobq.del +++ b/sjobq.del @@ -24,6 +24,32 @@ COUNTER_FILE="$DATA_DIR/counter" CURRENT_JOB_HOME="$DATA_DIR/current" MAX_HISTORY="10" +## +# @brief +## +diffTime() +{ + local time0=`echo $1 | sed 's/@/ /g'` + local time1=`echo $2 | sed 's/@/ /g'` + + local DAYS="" + local HOURS="" + local MINUTES="" + local SECONDS="" + + # %F = full date, %T = %H:%M:%S, %N = nanoseconds, %Z = time zone. + + DAYS=$(( $(printf '%s' $(( $(date -u -d"$time0" +%s) - $(date -u -d"$time1" +%s)))) / 60 / 60 / 24 )) + time1=$(date -d"$time1 +$DAYS days" '+%F %T.%N %Z') + HOURS=$(( $(printf '%s' $(( $(date -u -d"$time0" +%s) - $(date -u -d"$time1" +%s)))) / 60 / 60 )) + time1=$(date -d"$time1 +$HOURS hours" '+%F %T.%N %Z') + MINUTES=$(( $(printf '%s' $(( $(date -u -d"$time0" +%s) - $(date -u -d"$time1" +%s)))) / 60 )) + time1=$(date -d"$time1 +$MINUTES minutes" '+%F %T.%N %Z') + SECONDS=$(printf '%s' $(( $(date -u -d"$time0" +%s) - $(date -u -d"$time1" +%s)))) + + printf "%02d-%02d:%02d:%02d\n" "$DAYS" "$HOURS" "$MINUTES" "$SECONDS" +} + ## # @brief ## @@ -38,17 +64,19 @@ function showJobs() if [ -n "`ps -A | awk -v id=$lastID '($1==id){print "1"}'`" ] then currentPID="`cat $CURRENT_JOB_HOME/pid`" - spentTime=$(( $(date +%s) - $(cat $CURRENT_JOB_HOME/beginTime) )) + beginTime=`date -u '+%F %T.%N %Z' | sed 's/ /@/g'` + endTime=$(cat $CURRENT_JOB_HOME/beginTime) pstree -p $currentPID > $CURRENT_JOB_HOME/tree pstree -p $currentPID | awk 'BEGIN{RS="[()]"}($0~/^[[:digit:]]+$/){printf $0" "}' | sed 's/$/\n/' > $CURRENT_JOB_HOME/pids - echo "-------------" - echo " Current job" - echo "-------------" + echo "+-------------+" + echo "| Current job |" + echo "+-------------+" + echo "" echo "pid = `cat $CURRENT_JOB_HOME/pid`" echo "command = `cat $CURRENT_JOB_HOME/com`" - echo "dir = `cat $CURRENT_JOB_HOME/pwd`" - echo "time spent = $((spentTime/3600))h $(((spentTime/60)%60))m $((spentTime%60))s" + echo "dir = `cat $CURRENT_JOB_HOME/pwd | sed 's/\/home\/'$USER'\//~\//g'`" + echo "time spent = `diffTime $beginTime $endTime`" echo "pids = `cat $CURRENT_JOB_HOME/pids`" echo "tree =" cat $CURRENT_JOB_HOME/tree | sed '{s/^/\t/g}' @@ -58,26 +86,79 @@ function showJobs() if [ -n "$ID_LIST" ] then - echo "-----------" - echo " Job queue" - echo "-----------" + maxCharInDirColumn=0 + for procID in $ID_LIST + do + nCharInDir=`cat $DATA_DIR/$procID.pwd | sed 's/\/home\/'$USER'\//~\//g' | wc -c` + if [ "$maxCharInDirColumn" -lt "$nCharInDir" ]; then maxCharInDirColumn=$nCharInDir; fi + done + if [ "$maxCharInDirColumn" -lt 20 ]; then maxCharInDirColumn=20; fi + + rowSeparator=`echo -n "+-------+"` + rowSeparator=$rowSeparator`echo "-" | awk '{for(i=0;i<='$maxCharInDirColumn';i++) printf($1)}'` + rowSeparator=$rowSeparator`echo "+--------------------"` + printf "%s\n" "+-------+" + echo "| QUEUE |" + echo $rowSeparator + printf "|%6s |%"$maxCharInDirColumn"s | %s\n" "id" "directory" "command" + printf "|%6s |%"$maxCharInDirColumn"s | %s\n" "" "" "" for procID in $ID_LIST do - echo "id = `cat $DATA_DIR/$procID.bid`" - echo "command = `cat $DATA_DIR/$procID.com`" - echo "dir = `cat $DATA_DIR/$procID.pwd`" - echo "" + id=`cat $DATA_DIR/$procID.bid` + dir=`cat $DATA_DIR/$procID.pwd | sed 's/\/home\/'$USER'\//~\//g'` + command=`cat $DATA_DIR/$procID.com` + + printf "|%6s |%"$maxCharInDirColumn"s | " $id $dir + echo $command done + + echo $rowSeparator fi + echo "" if [ -f "$DATA_DIR/history" ] then - echo "---------" - echo " History" - echo "---------" + maxCharInDirColumn=`grep -E "dir\s+=" $DATA_DIR/history | sed 's/\/home\/'$USER'\//~\//g' \ + | sed -r 's/dir\s+=//g' | gawk '(length(line) Date: Thu, 8 Oct 2015 17:53:59 +0200 Subject: [PATCH 4/6] Create README.md --- README.md | 212 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..2074e11 --- /dev/null +++ b/README.md @@ -0,0 +1,212 @@ +SJobQ is a very simple queue system written in Bourne shell for managing serial executions running in a local form on only one machine. In some situations is important to let that a set of jobs running one by one without human supervision. This allow you to exploit the computing resources available on your machine, in order to keep the machine working all the time. The principal problem is that the available queue systems are a bit complicated to install or simply you need to be root to install them and you don't have those permissions. SJobQ is the solution, because, you don't need to be root and in order to install it you only need download three files and go !, you can begin to work. + +### INSTALLING +``` +DOWNLOAD THE .tar.gz FILE FROM THIS PAGE AND TYPE NEXT COMMANDS +$ tar xvfz nfaguirrec-sjobq-xxxxxxx.tar.gz +nfaguirrec-sjobq-xxxxxxx/ +nfaguirrec-sjobq-xxxxxxx/sjobq.d +nfaguirrec-sjobq-xxxxxxx/sjobq.del +nfaguirrec-sjobq-xxxxxxx/sjobq.push +nfaguirrec-sjobq-xxxxxxx/sjobq.stat + +$ ls nfaguirrec-sjobq-xxxxxxx +sjobq.d sjobq.del sjobq.push sjobq.stat + +$ cp nfaguirrec-sjobq-xxxxxxx/* $HOME/bin/ +``` + +### DEMON STARTUP +``` +$ sjobq.d start +============================== + SJobQ daemon has been started +============================== +``` + +### PUTTING JOBS IN THE QUEUE +To put jobs into queue, use the command sjob.queue before its statement. Its important to point out you have to use the character \ if you are going to use special bash characters like !,$,#. + +``` +$ sjobq.push sleep 30s +id = 1 +command = sleep 30s +dir = /home/nestor/Downloads + +$ sjobq.push echo \"Hola\" +id = 2 +command = echo "Hola" +dir = /home/nestor/Downloads + +$ sjobq.push find /etc/ -name \"\*.conf\" \> output \; sleep 30 +id = 3 +command = find /etc/ -name "*.conf" > output ; sleep 30 +dir = /home/nestor/Downloads +``` + +### CHECKING JOBS STATUS +The command sjobq.stat will show the jobs which are running, queued and finished. + +``` +$ sjobq.stat ++-------------+ +| Current job | ++-------------+ + +pid = 29406 +command = sleep 30s +dir = ~/Downloads +time spent = 00-00:00:09 +pids = 29406 +tree = + sleep(29406) + ++-------+ +| QUEUE | ++-------+------------------+---------- +| id | directory | command +| | | +| 2 | ~/Downloads | echo "Hola" +| 3 | ~/Downloads | find /etc/ -name "*.conf" > output ; sleep 30 ++-------+------------------+---------- +``` +after 2 minutes ... +``` +$ sjobq.stat + ++-------------+ +| HISTORY | ++-------------+------------------+---------- +| time spent | directory | command +| | | +| 00-00:00:30 | ~/Downloads | sleep 30s +| 00-00:00:04 | ~/Downloads | echo "Hola" +| 00-00:00:31 | ~/Downloads | find /etc/ -name "*.conf" > output ; sleep 30 ++-------------+------------------+---------- +``` + +### DELETING JOBS +Imagine that you put the following commands in the queue +``` +$ sjobq.push sleep 1h +id = 4 +command = sleep 1h +dir = ~/Downloads + +$ sjobq.push sleep 2h +id = 5 +command = sleep 1h +dir = ~/Downloads + +$ sjobq.push sleep 3h +id = 6 +command = sleep 1h +dir = ~/Downloads + +$ sjobq.stat ++-------------+ +| Current job | ++-------------+ + +pid = 30778 +command = sleep 1h +dir = ~/Downloads +time spent = 00-00:00:29 +pids = 30778 +tree = + sleep(30778) + ++-------+ +| QUEUE | ++-------+------------------+---------- +| id | directory | command +| | | +| 5 | ~/Downloads | sleep 2h +| 6 | ~/Downloads | sleep 3h ++-------+------------------+---------- + ++-------------+ +| HISTORY | ++-------------+------------------+---------- +| time spent | directory | command +| | | +| 00-00:00:30 | ~/Downloads | sleep 30s +| 00-00:00:04 | ~/Downloads | echo "Hola" +| 00-00:00:31 | ~/Downloads | find /etc/ -name "*.conf" > output ; sleep 30 ++-------------+------------------+---------- +``` +You can delete a queued job by using its identifier (id). The id is provided in the output of the sqjob.stat command. +``` +$ sjobq.del 6 +Job with id=6 has been deleted !! + +$ sjobq.stat ++-------------+ +| Current job | ++-------------+ + +pid = 30778 +command = sleep 1h +dir = ~/Downloads +time spent = 00-00:00:29 +pids = 30778 +tree = + sleep(30778) + ++-------+ +| QUEUE | ++-------+------------------+---------- +| id | directory | command +| | | +| 5 | ~/Downloads | sleep 2h ++-------+------------------+---------- + ++-------------+ +| HISTORY | ++-------------+------------------+---------- +| time spent | directory | command +| | | +| 00-00:00:30 | ~/Downloads | sleep 30s +| 00-00:00:04 | ~/Downloads | echo "Hola" +| 00-00:00:31 | ~/Downloads | find /etc/ -name "*.conf" > output ; sleep 30 ++-------------+------------------+---------- +``` +Also it is possible to delete the running job by using the identifier "current". +``` +$ sjobq.del current +Job with id=current has been deleted !! + +$ sjobq.stat ++-------------+ +| Current job | ++-------------+ + +pid = 31544 +command = sleep 2h +dir = ~/Downloads +time spent = 00-00:00:05 +pids = 31544 +tree = + sleep(31544) + ++-------------+ +| HISTORY | ++-------------+------------------+---------- +| time spent | directory | command +| | | +| 00-00:00:30 | ~/Downloads | sleep 30s +| 00-00:00:04 | ~/Downloads | echo "Hola" +| 00-00:00:31 | ~/Downloads | find /etc/ -name "*.conf" > output ; sleep 30 +| 00-00:02:57 | ~/Downloads | sleep 1h ++-------------+------------------+---------- +``` +### DAEMON SHUTDOWN +``` +$ sjobq.d stop +============================== + SJobQ daemon has been stopped +============================== +``` + +### Authors and Contributors +Néstor F. Aguirre (@nfaguirrec) 2010-2015 From 63251b79e2c06b517f52996ac8a088395250f58e Mon Sep 17 00:00:00 2001 From: nfaguirrec Date: Tue, 13 Oct 2015 17:28:56 +0200 Subject: [PATCH 5/6] bug fixed about formatting in HISTORY block --- sjobq.del | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sjobq.del b/sjobq.del index 802f222..2d95189 100755 --- a/sjobq.del +++ b/sjobq.del @@ -119,8 +119,11 @@ function showJobs() if [ -f "$DATA_DIR/history" ] then - maxCharInDirColumn=`grep -E "dir\s+=" $DATA_DIR/history | sed 's/\/home\/'$USER'\//~\//g' \ - | sed -r 's/dir\s+=//g' | gawk '(length(line) Date: Sun, 22 Nov 2015 00:14:18 +0100 Subject: [PATCH 6/6] Update sjobq.d With this change, now it is possible to execute commands into a NTFS filesystems --- sjobq.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sjobq.d b/sjobq.d index 150005f..d391d87 100755 --- a/sjobq.d +++ b/sjobq.d @@ -100,7 +100,7 @@ function update() echo "echo \$! > $CURRENT_JOB_HOME/pid" >> .tmp2156456 chmod +x .tmp2156456 - ./.tmp2156456 + bash ./.tmp2156456 rm .tmp2156456 rm -f $DATA_DIR/$IDL.bid