test scripts: job task can print stdout from file, shebangs fixes

examples/test scripts/Renders.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/bash
 
 RENDER="afrender"
 export AF_USERNAME="renderer"

examples/test scripts/clear.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/bash
 
 echo "Clearing tests in '$PWD'"
 

examples/test scripts/example.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import af

examples/test scripts/job.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 '''
 /* ''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' *\
@@ -52,6 +53,7 @@
 parser.add_option('-f', '--filesout',     dest='filesout',     type='string', default=None, help='Tasks out file [render/img.%04d.jpg]')
 parser.add_option(      '--filemin',      dest='filemin',      type='int',    default=-1, help='tasks output file size min')
 parser.add_option(      '--filemax',      dest='filemax',      type='int',    default=-1, help='tasks output file size max')
+parser.add_option(      '--stdoutfile',   dest='stdoutfile',   type='string', default=None, help='Read tasks stdout from file')
 parser.add_option(      '--mhmin',        dest='mhmin',        type='int',    default=-1, help='multi host tasks min hosts')
 parser.add_option(      '--mhmax',        dest='mhmax',        type='int',    default=-1, help='multi host tasks max hosts')
 parser.add_option(      '--mhwaitmax',    dest='mhwaitmax',    type='int',    default=0,  help='multi host tasks max hosts wait time seconds')
@@ -205,53 +207,52 @@
             else:
                 'Warning: Invalid tickets: "%s"' % Options.environment
 
-    str_capacity = ''
-    if Options.capmin != -1 or Options.capmax != -1:
-        block.setVariableCapacity(Options.capmin, Options.capmax)
-        str_capacity = ' -c ' + services.service.str_capacity
-
     if Options.filemin != -1 or Options.filemax != -1:
         block.setFileSizeCheck(Options.filemin, Options.filemax)
 
-    str_hosts = ''
-    if Options.mhmin != -1 or Options.mhmax != -1:
-        block.setMultiHost(
-            Options.mhmin, Options.mhmax, Options.mhwaitmax,
-            Options.mhsame, Options.mhservice, Options.mhwaitsrv
-        )
-        if Options.mhignorelost:
-            block.setSlaveLostIgnore()
-        str_hosts = ' ' + services.service.str_hosts
-
     negative_pertask = False
     if Options.frames != '':
         fr = frames[b].split('/')
         if int(fr[2]) < 0:
             negative_pertask = True
 
-    if not Options.stringtype and not negative_pertask:
-        cmd = 'task.py'
-        cmd = "\"%s\"" % os.path.join(os.getcwd(), cmd)
-        cmd = "%s %s" % (os.getenv('CGRU_PYTHONEXE','python'), cmd)
-        cmd += ' --exitstatus %d ' % Options.exitstatus
-
-        if Options.filesout:
-            cmd += ' --filesout "%s"' % Options.filesout
-            block.skipExistingFiles()
-            block.checkRenderedFiles(100)
-            files = []
-            for afile in Options.filesout.split(';'):
-                files.append(afcommon.patternFromStdC(afile))
-            block.setFiles(files)
+    cmd = 'task.py'
+    cmd = "\"%s\"" % os.path.join(os.getcwd(), cmd)
+    cmd = "%s %s" % (os.getenv('CGRU_PYTHONEXE','python3'), cmd)
+    cmd += ' --exitstatus %d ' % Options.exitstatus
 
-        cmd += '%(str_capacity)s%(str_hosts)s -s @#@ -e @#@ ' \
-               '-i %(increment)d -t %(timesec)g -r %(randtime)g --pkp %(pkp)d ' \
-               '-v %(verbose)d @####@ @#####@ @#####@ @#####@' % vars()
+    if Options.capmin != -1 or Options.capmax != -1:
+        block.setVariableCapacity(Options.capmin, Options.capmax)
+        cmd += ' -c ' + services.service.str_capacity
 
-        if Options.cmd:
-            cmd = Options.cmd
+    if Options.mhmin != -1 or Options.mhmax != -1:
+        block.setMultiHost(
+            Options.mhmin, Options.mhmax, Options.mhwaitmax,
+            Options.mhsame, Options.mhservice, Options.mhwaitsrv
+        )
+        if Options.mhignorelost:
+            block.setSlaveLostIgnore()
+        cmd += ' ' + services.service.str_hosts
+
+    if Options.filesout:
+        cmd += ' --filesout "%s"' % Options.filesout
+        block.skipExistingFiles()
+        block.checkRenderedFiles(100)
+        files = []
+        for afile in Options.filesout.split(';'):
+            files.append(afcommon.patternFromStdC(afile))
+        block.setFiles(files)
+
+    if Options.stdoutfile:
+        if not os.path.isfile(Options.stdoutfile):
+            print('ERROR: File "%s" does not exist.')
+            sys.exit(1)
+        cmd += ' --stdoutfile "%s"' % Options.stdoutfile
 
-        block.setCommand(cmd, False)
+    if not Options.stringtype and not negative_pertask:
+        cmd += ' -s @#@ -e @#@ -i %(increment)d' \
+               ' -t %(timesec)g -r %(randtime)g --pkp %(pkp)d' \
+               ' -v %(verbose)d @####@ @#####@ @#####@ @#####@' % vars()
 
         if Options.frames != '':
             fr = frames[b].split('/')
@@ -260,25 +261,11 @@
             block.setNumeric(1, numtasks, Options.pertask, increment)
 
     else:
-        cmd = 'task.py%(str_capacity)s @#@ -v %(verbose)d' % vars()
-        cmd = "%s %s" % (os.getenv('CGRU_PYTHONEXE','python'), cmd)
+        cmd += ' -v %(verbose)d' % vars()
+        cmd += ' @#@'
 
         block.setTasksName('task @#@')
 
-        if Options.filesout:
-            cmd += ' --filesout "%s"' % Options.filesout
-            files = []
-            for afile in Options.filesout.split(';'):
-                files.append(afile.replace('%04d','@#@'))
-            block.setFiles(files)
-            block.skipExistingFiles()
-            block.checkRenderedFiles(100)
-
-        if Options.cmd:
-            cmd = Options.cmd
-
-        block.setCommand( cmd, False)
-
         if Options.frames != '':
             fr = frames[b].split('/')
             block.setFramesPerTask(int(fr[2]))
@@ -294,6 +281,11 @@
 
             block.tasks.append(task)
 
+    if Options.cmd:
+        cmd = Options.cmd
+
+    block.setCommand(cmd, False)
+
 if Options.cmdpre != '':
     job.setCmdPre(Options.cmdpre)
 if Options.cmdpost != '':

examples/test scripts/job.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/bash
 
 source ./setup.sh
 

examples/test scripts/multihost.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/bash
 
 ./job.sh --pkp 0 --mhmin 3 --mhmax 5 --mhwaitmax 5 --mhignorelost 1 "$@"
 

examples/test scripts/setup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/bash
 
 pushd ../.. >> /dev/null
 source ./setup.sh

examples/test scripts/setup_pgsql.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/bash
 
 pg_conf=`find /etc/postgresql -name "pg_hba.conf"`
 pg_user="postgres"

examples/test scripts/subtask.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import os

examples/test scripts/task.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import os
@@ -7,7 +8,8 @@
 import time
 
 
-print('Started at ' + time.strftime('%A %d %B %H:%M:%S'))
+time_start = time.time()
+print('Started at: %s' % time.ctime(time_start))
 print('COMMAND:')
 print(sys.argv)
 print('WORKING DIRECTORY:')
@@ -35,6 +37,7 @@ def interrupt(signum, frame):
 Parser.add_option('-t', '--time',      dest='timesec',   type='float',  default=2,    help='Time per frame in seconds')
 Parser.add_option('-r', '--randtime',  dest='randtime',  type='float',  default=0,    help='Random time per frame in seconds')
 Parser.add_option('-f', '--filesout',  dest='filesout',  type='string', default=None, help='File(s) to write (";" separated)')
+Parser.add_option(      '--stdoutfile',dest='stdoutfile',type='string', default=None, help='File to read stdout from')
 Parser.add_option('-v', '--verbose',   dest='verbose',   type='int',    default=0,    help='Verbose')
 Parser.add_option('-p', '--pkp',       dest='pkp',       type='int',    default=1,    help='Parser key percentage')
 Parser.add_option('-H', '--hosts',     dest='hosts',     type='string', default=None, help='Hosts list for mutihost tasks')
@@ -49,6 +52,15 @@ def interrupt(signum, frame):
 ParserKeys = ['[ PARSER WARNING ]', '[ PARSER ERROR ]',
               '[ PARSER BAD RESULT ]', '[ PARSER FINISHED SUCCESS ]']
 
+StdOut = None
+if Options.stdoutfile:
+    if not os.path.isfile(Options.stdoutfile):
+        print('ERROR: File "%s" does not exist.')
+        sys.exit(1)
+    print('Reading stdout from "%s"' % Options.stdoutfile)
+    with open(Options.stdoutfile) as file:
+        StdOut = file.readlines()
+
 # Check frame range settings:
 if frame_end < frame_start:
     print('Error: frame_end(%d) < frame_start(%d)' % (frame_end, frame_start))
@@ -60,46 +72,46 @@ def interrupt(signum, frame):
     frame_inc = 1
     print('[ PARSER WARNING ]')
 
-sleepsec = .01 * (Options.timesec + Options.randtime * random.random())
-
 frame = frame_start
 parserKey_CurIndex = int(random.random() * 100) % len(ParserKeys)
 
-time_start = time.time()
-print('Started at: %s' % time.ctime(time_start))
-
 while frame <= frame_end:
     print('FRAME: %s' % frame)
 
-    for p in range(100):
-        print('PROGRESS: {progress}%'.format(progress=p + 1))
+    if StdOut is None:
+        for p in range(100):
+            print('PROGRESS: {progress}%'.format(progress=p + 1))
+
+            if p == 10:
+                print('ACTIVITY: Generating')
 
-        if p == 10:
-            print('ACTIVITY: Generating')
+            if p == 50:
+                print('ACTIVITY: Rendering')
+                print('REPORT: ' + str(random.random()))
 
-        if p == 50:
-            print('ACTIVITY: Rendering')
-            print('REPORT: ' + str(random.random()))
+            if p == 90:
+                print('ACTIVITY: Finalizing')
 
-        if p == 90:
-            print('ACTIVITY: Finalizing')
+            if random.random() * 100 * 100 < Options.pkp:
+                print(ParserKeys[parserKey_CurIndex])
+                parserKey_CurIndex += 1
+                if parserKey_CurIndex >= len(ParserKeys):
+                    parserKey_CurIndex = 0
 
-        if random.random() * 100 * 100 < Options.pkp:
-            print(ParserKeys[parserKey_CurIndex])
-            parserKey_CurIndex += 1
-            if parserKey_CurIndex >= len(ParserKeys):
-                parserKey_CurIndex = 0
+            for v in range(Options.verbose):
+                print(
+                    '%s: %s: %s: QWERTYUIOPASDFGHJKLZXCVBNM1234567890qwertyuiopasd'
+                    'fghjklzxcvbnm' % (frame, p, v)
+                )
 
-        for v in range(Options.verbose):
-            print(
-                '%s: %s: %s: QWERTYUIOPASDFGHJKLZXCVBNM1234567890qwertyuiopasd'
-                'fghjklzxcvbnm' % (frame, p, v)
-            )
-            # sys.stdout.flush()
-            # time.sleep(sleepsec)
+            sys.stdout.flush()
+            time.sleep(.01 * (Options.timesec + Options.randtime * random.random()))
 
-        sys.stdout.flush()
-        time.sleep(sleepsec)
+    else:
+        for line in StdOut:
+            print(line, end='')
+            sys.stdout.flush()
+            time.sleep(Options.timesec / len(StdOut))
 
     if Options.filesout:
         files = Options.filesout.split(';')
@@ -121,7 +133,6 @@ def interrupt(signum, frame):
 
 time_finish = time.time()
 print('Finished at: %s' % time.ctime(time_finish))
-print('Sleeping = %f seconds.' % sleepsec)
 print('Running time = %d seconds.' % (time_finish - time_start))
 
 sys.stdout.flush()