-
Notifications
You must be signed in to change notification settings - Fork 0
/
runbatchprocess.py
41 lines (36 loc) · 1.61 KB
/
runbatchprocess.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
"""Given the path to a list of files run a system process against that file.
Employs multiprocessing. Set the concurrentProcesses argument to determine the
number of processes to run at a time.
"""
import shutil, os
import glob
import subprocess
from string import Template
import logging
from multiprocessing import Pool
import math
def executeSystemProcesses(objFileName, commandTemplate):
objFileName = objFileName.strip()
logging.debug(objFileName)
objDirName = os.path.dirname(objFileName)
command = commandTemplate.substitute(objFileName=objFileName, objDirName=objDirName)
logging.debug(command)
subprocess.call(command, shell=True)
def process(FILE_LIST_FILENAME, commandTemplateString, concurrentProcesses=0):
"""Go through the list of files and run the provided command against them,
one at a time. Template string maps the terms $objFileName and $objDirName.
Example:
>>> runBatchProcess('convert -scale 256 "$objFileName" "$objDirName/TN.jpg"')
"""
commandTemplate = Template(commandTemplateString)
mpBatchMap = []
# Round number of procs down to the nearest integer
concurrentProcesses = math.trunc(concurrentProcesses)
with open(FILE_LIST_FILENAME) as fileList:
for line in fileList:
mpBatchMap.append((line, commandTemplate))
logging.debug('Starting MP batch of %i with %i concurrent processes.' % (len(mpBatchMap), concurrentProcesses))
if mpBatchMap:
with Pool(concurrentProcesses) as p:
poolResult = p.starmap(executeSystemProcesses, mpBatchMap)
logging.debug('Pool result: %s' % str(poolResult))