Andreas/evennia
1
0
Fork 0
mirror of https://github.com/evennia/evennia.git synced 2026-03-23 16:26:30 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added new process-pool runner based on AMPoule (integrated into Evennia).

Browse source 
This allows e.g. utils.utils.run_async to offload long-running functions
to a completely different subprocess entirely, offering real parallelism.

Implementation is still experimental, notably not all objects can be
transferred safely across the wire; also there is no concept of
updating caches yet - so adding an object from the subprocess side
will not be known in the main thread yet (since caches cannot yet tell
the underlying database has changed).
This commit is contained in:
Griatch 2012-09-02 10:10:22 +02:00
parent dcc7f29a91
commit f5a889e40c
22 changed files with 2322 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

95
src/commands/default/batchprocess.py
View file

@ -31,8 +31,8 @@ from src.utils import utils
# limit symbols for API inclusion
__all__ = ("CmdBatchCommands", "CmdBatchCode")
HEADER_WIDTH = 70
UTF8_ERROR = \
_HEADER_WIDTH = 70
_UTF8_ERROR = \
"""
{rDecode error in '%s'.{n
@ -50,6 +50,35 @@ UTF8_ERROR = \
The (first) error was found with a character on line %s in the file.
"""
_PROCPOOL_BATCHCMD_SOURCE = """
from src.commands.default.batchprocess import batch_cmd_exec, step_pointer, BatchSafeCmdSet
caller.ndb.batch_stack = commands
caller.ndb.batch_stackptr = 0
caller.ndb.batch_batchmode = "batch_commands"
caller.cmdset.add(BatchSafeCmdSet)
for inum in range(len(commands)):
print "command:", inum
caller.cmdset.add(BatchSafeCmdSet)
if not batch_cmd_exec(caller):
break
step_pointer(caller, 1)
print "leaving run ..."
"""
_PROCPOOL_BATCHCODE_SOURCE = """
from src.commands.default.batchprocess import batch_cmd_exec, step_pointer, BatchSafeCmdSet
caller.ndb.batch_stack = commands
caller.ndb.batch_stackptr = 0
caller.ndb.batch_batchmode = "batch_commands"
caller.cmdset.add(BatchSafeCmdSet)
for inum in range(len(commands)):
print "command:", inum
caller.cmdset.add(BatchSafeCmdSet)
if not batch_cmd_exec(caller):
break
step_pointer(caller, 1)
print "leaving run ..."
"""
#------------------------------------------------------------
# Helper functions
@ -59,7 +88,7 @@ def format_header(caller, entry):
"""
Formats a header
"""
width = HEADER_WIDTH - 10
width = _HEADER_WIDTH - 10
entry = entry.strip()
header = utils.crop(entry, width=width)
ptr = caller.ndb.batch_stackptr + 1
@ -215,7 +244,7 @@ class CmdBatchCommands(MuxCommand):
commands = BATCHCMD.parse_file(python_path)
except UnicodeDecodeError, err:
lnum = err.linenum
caller.msg(UTF8_ERROR % (python_path, lnum))
caller.msg(_UTF8_ERROR % (python_path, lnum))
return
if not commands:
@ -242,16 +271,24 @@ class CmdBatchCommands(MuxCommand):
show_curr(caller)
else:
caller.msg("Running Batch-command processor - Automatic mode for %s (this might take some time) ..." % python_path)
# add the 'safety' cmdset in case the batch processing adds cmdsets to us
for inum in range(len(commands)):
# loop through the batch file
if not batch_cmd_exec(caller):
return
step_pointer(caller, 1)
# clean out the safety cmdset and clean out all other temporary attrs.
string = " Batchfile '%s' applied." % python_path
caller.msg("{G%s" % string)
purge_processor(caller)
if settings.PROCPOOL_ENABLED:
# run in parallel process
def callback(r):
caller.msg(" {GBatchfile '%s' applied." % python_path)
purge_processor(caller)
utils.run_async(_PROCPOOL_BATCHCMD_SOURCE, commands=commands, caller=caller, at_return=callback)
else:
# run in-process (might block)
for inum in range(len(commands)):
# loop through the batch file
if not batch_cmd_exec(caller):
return
step_pointer(caller, 1)
# clean out the safety cmdset and clean out all other temporary attrs.
string = " Batchfile '%s' applied." % python_path
caller.msg("{G%s" % string)
purge_processor(caller)
class CmdBatchCode(MuxCommand):
"""
@ -293,7 +330,7 @@ class CmdBatchCode(MuxCommand):
codes = BATCHCODE.parse_file(python_path)
except UnicodeDecodeError, err:
lnum = err.linenum
caller.msg(UTF8_ERROR % (python_path, lnum))
caller.msg(_UTF8_ERROR % (python_path, lnum))
return
if not codes:
@ -325,15 +362,25 @@ class CmdBatchCode(MuxCommand):
show_curr(caller)
else:
caller.msg("Running Batch-code processor - Automatic mode for %s ..." % python_path)
# add the 'safety' cmdset in case the batch processing adds cmdsets to us
for inum in range(len(codes)):
# loop through the batch file
if not batch_code_exec(caller):
return
step_pointer(caller, 1)
string = " Batchfile '%s' applied." % python_path
caller.msg("{G%s" % string)
purge_processor(caller)
if settings.PROCPOOL_ENABLED:
# run in parallel process
def callback(r):
caller.msg(" {GBatchfile '%s' applied." % python_path)
purge_processor(caller)
utils.run_async(_PROCPOOL_BATCHCODE_SOURCE, commands=commands, caller=caller, at_return=callback)
else:
# un in-process (will block)
for inum in range(len(commands)):
# loop through the batch file
if not batch_cmd_exec(caller):
return
step_pointer(caller, 1)
# clean out the safety cmdset and clean out all other temporary attrs.
string = " Batchfile '%s' applied." % python_path
caller.msg("{G%s" % string)
purge_processor(caller)
#------------------------------------------------------------
# State-commands for the interactive batch processor modes