From 87b6cee59635f742cf66fca6b68a0e61bcb3a1bf Mon Sep 17 00:00:00 2001 From: Vincent Le Goff Date: Thu, 20 Jul 2017 21:37:34 +0200 Subject: [PATCH 1/4] Add the persistent utils.delay --- evennia/server/server.py | 5 ++ evennia/utils/persistent.py | 173 ++++++++++++++++++++++++++++++++++++ evennia/utils/utils.py | 24 ++++- 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 evennia/utils/persistent.py diff --git a/evennia/server/server.py b/evennia/server/server.py index 038846e5b4..7c4bf46056 100644 --- a/evennia/server/server.py +++ b/evennia/server/server.py @@ -455,6 +455,11 @@ class Evennia(object): # (this also starts any that didn't yet start) ScriptDB.objects.validate(init_mode=mode) + # start the persistent tasks + from evennia.utils.persistent import PERSISTENT_TASKS + PERSISTENT_TASKS.load() + PERSISTENT_TASKS.create_delays() + # delete the temporary setting ServerConfig.objects.conf("server_restart_mode", delete=True) diff --git a/evennia/utils/persistent.py b/evennia/utils/persistent.py new file mode 100644 index 0000000000..b0cd3275fd --- /dev/null +++ b/evennia/utils/persistent.py @@ -0,0 +1,173 @@ +""" +Module containing persistent features and the persistent differed +tasks, generated by utils.delay(persistent=True). + +""" + +from datetime import datetime, timedelta + +from twisted.internet import reactor, task +from evennia.server.models import ServerConfig +from evennia.utils.logger import log_trace, log_err +from evennia.utils.dbserialize import dbserialize, dbunserialize + +PERSISTENT_TASKS = None + +class PersistentTasks(object): + + """A light singleton wrapper allowing to access permanent tasks. + + Permament tasks are created by `utils.delay` when the `persistent` + keyword argument is set to True. Tasks are saved in ServerConfig. + It's easier to access these tasks (should it be necessary) using + `evennia.utils.persistent.PERSISTSENT_TASKS`, which contains one + instance of this class, and use its `add and `remove` methods. + + """ + + def __init__(self): + self.tasks = {} + + def load(self): + """Load from the ServerConfig. + + Note: + This should be automatically called when Evennia starts. + It populates `self.tasks` according to the ServerConfig. + + """ + value = ServerConfig.objects.conf("delayed_tasks", default={}) + if isinstance(value, basestring): + tasks = dbunserialize(value) + else: + tasks = value + + # At this point, `tasks` contains a unserialized dictionary of tasks + for task_id, (date, callback, args, kwargs) in tasks.items(): + if isinstance(callback, tuple): + # `callback` can be an object and name for instance methods + obj, method = callback + callback = getattr(obj, method) + self.tasks[task_id] = (date, callback, args, kwargs) + + def save(self): + """Save the tasks in ServerConfig.""" + to_save = {} + for task_id, (date, callback, args, kwargs) in self.tasks.items(): + if getattr(callback, "__self__", None): + # `callback` is an instance method + obj = callback.__self__ + name = callback.__name__ + callback = (obj, name) + + # Check if callback can be pickled. args and kwargs have been checked + safe_callback = None + + try: + dbserialize(callback) + except (TypeError, AttributeError): + raise ValueError("the specified callback {} cannot be pickled. " \ + "It must be a top-level function in a module or an " \ + "instance method.".format(callback)) + else: + safe_callback = callback + + to_save[task_id] = (date, safe_callback, args, kwargs) + to_save = dbserialize(to_save) + ServerConfig.objects.conf("delayed_tasks", to_save) + + def add(self, timedelay, callback, *args, **kwargs): + """Add a new persistent task in the configuration. + + Args: + timedelay (int or float): time in sedconds before calling the callback. + callback (function or instance method): the callback itself + any (any): any additional positional arguments to send to the callback + + Kwargs: + any (any): additional keyword arguments to send to the callback + + Note: + This doesn't create any delay at this time to call the task, + but the task is written in ServerConfig along with the + others. + + """ + now = datetime.now() + delta = timedelta(seconds=timedelay) + + # Choose a free task_id + safe_args = [] + safe_kwargs = {} + used_ids = self.tasks.keys() + task_id = 1 + while task_id in used_ids: + task_id += 1 + + # Check that args and kwargs contain picklable information + for arg in args: + try: + dbserialize(arg) + except (TypeError, AttributeError): + logger.log_err("The positional argument {} cannot be " \ + "pickled and will not be present in the arguments " \ + "fed to the callback {}".format(arg, callback)) + else: + safe_args.append(arg) + + for key, value in kwargs.items(): + try: + dbserialize(value) + except (TypeError, AttributeError): + logger.log_err("The {} keyword argument {} cannot be " \ + "pickled and will not be present in the arguments " \ + "fed to the callback {}".format(key, value, callback)) + else: + safe_kwargs[key] = value + + self.tasks[task_id] = (now + delta, callback, safe_args, safe_kwargs) + self.save() + return task_id + + def remove(self, task_id): + """Remove a task without executing it. + + Args: + task_id (int): an existing task ID. + + """ + del self.tasks[task_id] + self.save() + + def do_task(self, task_id): + """Execute the task (call its callback). + + Args: + task_id (int): a valid task ID. + + Note: + This will also remove it from the list of current tasks. + + """ + date, callback, args, kwargs = self.tasks.pop(task_id) + self.save() + callback(*args, **kwargs) + + def create_delays(self): + """Create the delayed tasks for the persistent tasks. + + Note: + This method should be automatically called when Evennia starts. + + """ + now = datetime.now() + for task_id, (date, callbac, args, kwargs) in self.tasks.items(): + seconds = (date - now).total_seconds() + if seconds < 0: + seconds = 0 + + task.deferLater(reactor, seconds, self.do_task, task_id) + + +# Create the soft singleton +PERSISTENT_TASKS = PersistentTasks() diff --git a/evennia/utils/utils.py b/evennia/utils/utils.py index 05ff24f0ed..a7f47ca555 100644 --- a/evennia/utils/utils.py +++ b/evennia/utils/utils.py @@ -44,7 +44,6 @@ _DA = object.__delattr__ _DEFAULT_WIDTH = settings.CLIENT_DEFAULT_WIDTH - def is_iter(iterable): """ Checks if an object behaves iterably. @@ -920,6 +919,8 @@ def uses_database(name="sqlite3"): return engine == "django.db.backends.%s" % name +_PERSISTENT_TASKS = None + def delay(timedelay, callback, *args, **kwargs): """ Delay the return of a value. @@ -930,6 +931,8 @@ def delay(timedelay, callback, *args, **kwargs): arguments after `timedelay` seconds. args (any, optional): Will be used as arguments to callback Kwargs: + persistent (bool, optional): should make the delay persistent + over a reboot or reload any (any): Will be used to call the callback. Returns: @@ -939,7 +942,26 @@ def delay(timedelay, callback, *args, **kwargs): defined directly in the command body and don't need to be specified here. + Note: + If persistent is set to True, the callback, its arguments + and other keyword arguments will be saved in the database, + assuming they can be. The callback will be executed even after + a server restart/reload, taking into account the specified delay + (and server down time). + """ + global _PERSISTENT_TASKS + persistent = kwargs.get("persistent", False) + if persistent: + del kwargs["persistent"] + # Do some imports here to avoid circular import and speed things up + if _PERSISTENT_TASKS is None: + from evennia.utils.persistent import PERSISTENT_TASKS as _PERSISTENT_TASKS + task_id = _PERSISTENT_TASKS.add(timedelay, callback, *args, **kwargs) + callback = _PERSISTENT_TASKS.do_task + args = [task_id] + kwargs = {} + return task.deferLater(reactor, timedelay, callback, *args, **kwargs) From e0eb490814f9b9cf7e05fe8f5444c49ab71d9ca1 Mon Sep 17 00:00:00 2001 From: Vincent Le Goff Date: Fri, 21 Jul 2017 18:38:11 +0200 Subject: [PATCH 2/4] Slightly optimize persistent tasks and serialization --- evennia/utils/persistent.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/evennia/utils/persistent.py b/evennia/utils/persistent.py index b0cd3275fd..5fdac2ceff 100644 --- a/evennia/utils/persistent.py +++ b/evennia/utils/persistent.py @@ -27,6 +27,7 @@ class PersistentTasks(object): def __init__(self): self.tasks = {} + self.to_save = {} def load(self): """Load from the ServerConfig. @@ -42,8 +43,9 @@ class PersistentTasks(object): else: tasks = value - # At this point, `tasks` contains a unserialized dictionary of tasks - for task_id, (date, callback, args, kwargs) in tasks.items(): + # At this point, `tasks` contains a dictionary of still-serialized tasks + for task_id, value in tasks.items(): + date, callback, args, kwargs = dbunserialize(value) if isinstance(callback, tuple): # `callback` can be an object and name for instance methods obj, method = callback @@ -52,8 +54,10 @@ class PersistentTasks(object): def save(self): """Save the tasks in ServerConfig.""" - to_save = {} for task_id, (date, callback, args, kwargs) in self.tasks.items(): + if task_id in self.to_save: + continue + if getattr(callback, "__self__", None): # `callback` is an instance method obj = callback.__self__ @@ -72,9 +76,8 @@ class PersistentTasks(object): else: safe_callback = callback - to_save[task_id] = (date, safe_callback, args, kwargs) - to_save = dbserialize(to_save) - ServerConfig.objects.conf("delayed_tasks", to_save) + self.to_save[task_id] = dbserialize((date, safe_callback, args, kwargs)) + ServerConfig.objects.conf("delayed_tasks", self.to_save) def add(self, timedelay, callback, *args, **kwargs): """Add a new persistent task in the configuration. @@ -137,6 +140,9 @@ class PersistentTasks(object): """ del self.tasks[task_id] + if task_id in self.to_save: + del self.to_save[task_id] + self.save() def do_task(self, task_id): @@ -150,6 +156,9 @@ class PersistentTasks(object): """ date, callback, args, kwargs = self.tasks.pop(task_id) + if task_id in self.to_save: + del self.to_save[task_id] + self.save() callback(*args, **kwargs) From 37c9d65a9de531ea0318e2d8c2daac6a7be610f3 Mon Sep 17 00:00:00 2001 From: Vincent Le Goff Date: Tue, 8 Aug 2017 19:47:22 +0200 Subject: [PATCH 3/4] Place the persistent task in a task handler --- .../persistent.py => scripts/taskhandler.py} | 105 ++++++++++-------- evennia/server/server.py | 8 +- evennia/utils/utils.py | 28 ++--- 3 files changed, 72 insertions(+), 69 deletions(-) rename evennia/{utils/persistent.py => scripts/taskhandler.py} (62%) diff --git a/evennia/utils/persistent.py b/evennia/scripts/taskhandler.py similarity index 62% rename from evennia/utils/persistent.py rename to evennia/scripts/taskhandler.py index 5fdac2ceff..0fc3bfe732 100644 --- a/evennia/utils/persistent.py +++ b/evennia/scripts/taskhandler.py @@ -1,7 +1,5 @@ """ -Module containing persistent features and the persistent differed -tasks, generated by utils.delay(persistent=True). - +Module containing the task handler for Evennia deferred tasks, persistent or not. """ from datetime import datetime, timedelta @@ -11,17 +9,20 @@ from evennia.server.models import ServerConfig from evennia.utils.logger import log_trace, log_err from evennia.utils.dbserialize import dbserialize, dbunserialize -PERSISTENT_TASKS = None +TASK_HANDLER = None -class PersistentTasks(object): +class TaskHandler(object): - """A light singleton wrapper allowing to access permanent tasks. + """ + A light singleton wrapper allowing to access permanent tasks. + + When `utils.delay` is called, the task handler is used to create + the task. If `utils.delay` is called with `persistent=True`, the + task handler stores the new task and saves. - Permament tasks are created by `utils.delay` when the `persistent` - keyword argument is set to True. Tasks are saved in ServerConfig. It's easier to access these tasks (should it be necessary) using - `evennia.utils.persistent.PERSISTSENT_TASKS`, which contains one - instance of this class, and use its `add and `remove` methods. + `evennia.scripts.taskhandler.TASK_HANDLER`, which contains one + instance of this class, and use its `add` and `remove` methods. """ @@ -88,56 +89,63 @@ class PersistentTasks(object): any (any): any additional positional arguments to send to the callback Kwargs: + persistent (bool, optional): persist the task (store it). any (any): additional keyword arguments to send to the callback - Note: - This doesn't create any delay at this time to call the task, - but the task is written in ServerConfig along with the - others. - """ - now = datetime.now() - delta = timedelta(seconds=timedelay) + persistent = kwargs.get("persistent", False) + if persistent: + del kwargs["persistent"] + now = datetime.now() + delta = timedelta(seconds=timedelay) - # Choose a free task_id - safe_args = [] - safe_kwargs = {} - used_ids = self.tasks.keys() - task_id = 1 - while task_id in used_ids: - task_id += 1 + # Choose a free task_id + safe_args = [] + safe_kwargs = {} + used_ids = self.tasks.keys() + task_id = 1 + while task_id in used_ids: + task_id += 1 - # Check that args and kwargs contain picklable information - for arg in args: - try: - dbserialize(arg) - except (TypeError, AttributeError): - logger.log_err("The positional argument {} cannot be " \ - "pickled and will not be present in the arguments " \ - "fed to the callback {}".format(arg, callback)) - else: - safe_args.append(arg) + # Check that args and kwargs contain picklable information + for arg in args: + try: + dbserialize(arg) + except (TypeError, AttributeError): + logger.log_err("The positional argument {} cannot be " \ + "pickled and will not be present in the arguments " \ + "fed to the callback {}".format(arg, callback)) + else: + safe_args.append(arg) - for key, value in kwargs.items(): - try: - dbserialize(value) - except (TypeError, AttributeError): - logger.log_err("The {} keyword argument {} cannot be " \ - "pickled and will not be present in the arguments " \ - "fed to the callback {}".format(key, value, callback)) - else: - safe_kwargs[key] = value + for key, value in kwargs.items(): + try: + dbserialize(value) + except (TypeError, AttributeError): + logger.log_err("The {} keyword argument {} cannot be " \ + "pickled and will not be present in the arguments " \ + "fed to the callback {}".format(key, value, callback)) + else: + safe_kwargs[key] = value - self.tasks[task_id] = (now + delta, callback, safe_args, safe_kwargs) - self.save() - return task_id + self.tasks[task_id] = (now + delta, callback, safe_args, safe_kwargs) + self.save() + callback = self.do_task + args = [task_id] + kwargs = {} + + return task.deferLater(reactor, timedelay, callback, *args, **kwargs) def remove(self, task_id): - """Remove a task without executing it. + """Remove a persistent task without executing it. Args: task_id (int): an existing task ID. + Note: + A non-persistent task doesn't have a task_id, it is not stored + in the TaskHandler. + """ del self.tasks[task_id] if task_id in self.to_save: @@ -179,4 +187,5 @@ class PersistentTasks(object): # Create the soft singleton -PERSISTENT_TASKS = PersistentTasks() +TASK_HANDLER = TaskHandler() + diff --git a/evennia/server/server.py b/evennia/server/server.py index 7c4bf46056..d6a47c208b 100644 --- a/evennia/server/server.py +++ b/evennia/server/server.py @@ -455,10 +455,10 @@ class Evennia(object): # (this also starts any that didn't yet start) ScriptDB.objects.validate(init_mode=mode) - # start the persistent tasks - from evennia.utils.persistent import PERSISTENT_TASKS - PERSISTENT_TASKS.load() - PERSISTENT_TASKS.create_delays() + # start the task handler + from evennia.scripts.taskhandler import TASK_HANDLER + TASK_HANDLER.load() + TASK_HANDLER.create_delays() # delete the temporary setting ServerConfig.objects.conf("server_restart_mode", delete=True) diff --git a/evennia/utils/utils.py b/evennia/utils/utils.py index a7f47ca555..ab67421ad4 100644 --- a/evennia/utils/utils.py +++ b/evennia/utils/utils.py @@ -919,7 +919,7 @@ def uses_database(name="sqlite3"): return engine == "django.db.backends.%s" % name -_PERSISTENT_TASKS = None +_TASK_HANDLER = None def delay(timedelay, callback, *args, **kwargs): """ @@ -931,9 +931,9 @@ def delay(timedelay, callback, *args, **kwargs): arguments after `timedelay` seconds. args (any, optional): Will be used as arguments to callback Kwargs: - persistent (bool, optional): should make the delay persistent - over a reboot or reload - any (any): Will be used to call the callback. + persistent (bool, optional): should make the delay persistent + over a reboot or reload + any (any): Will be used to call the callback. Returns: deferred (deferred): Will fire fire with callback after @@ -943,6 +943,8 @@ def delay(timedelay, callback, *args, **kwargs): specified here. Note: + The task handler (`evennia.scripts.taskhandler.TASK_HANDLEr`) will + be called for persistent or non-persistent tasks. If persistent is set to True, the callback, its arguments and other keyword arguments will be saved in the database, assuming they can be. The callback will be executed even after @@ -950,19 +952,11 @@ def delay(timedelay, callback, *args, **kwargs): (and server down time). """ - global _PERSISTENT_TASKS - persistent = kwargs.get("persistent", False) - if persistent: - del kwargs["persistent"] - # Do some imports here to avoid circular import and speed things up - if _PERSISTENT_TASKS is None: - from evennia.utils.persistent import PERSISTENT_TASKS as _PERSISTENT_TASKS - task_id = _PERSISTENT_TASKS.add(timedelay, callback, *args, **kwargs) - callback = _PERSISTENT_TASKS.do_task - args = [task_id] - kwargs = {} - - return task.deferLater(reactor, timedelay, callback, *args, **kwargs) + global _TASK_HANDLER + # Do some imports here to avoid circular import and speed things up + if _TASK_HANDLER is None: + from evennia.scripts.taskhandler import TASK_HANDLER as _TASK_HANDLER + return _TASK_HANDLER.add(timedelay, callback, *args, **kwargs) _TYPECLASSMODELS = None From 8ef50f3706c82cbedf64a069fbd0a65a6e2885d2 Mon Sep 17 00:00:00 2001 From: Vincent Le Goff Date: Thu, 10 Aug 2017 11:40:57 +0200 Subject: [PATCH 4/4] Fix a minor mistake in the task handler --- evennia/scripts/taskhandler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/evennia/scripts/taskhandler.py b/evennia/scripts/taskhandler.py index 0fc3bfe732..aadc3d9746 100644 --- a/evennia/scripts/taskhandler.py +++ b/evennia/scripts/taskhandler.py @@ -179,10 +179,7 @@ class TaskHandler(object): """ now = datetime.now() for task_id, (date, callbac, args, kwargs) in self.tasks.items(): - seconds = (date - now).total_seconds() - if seconds < 0: - seconds = 0 - + seconds = max(0, (date - now).total_seconds()) task.deferLater(reactor, seconds, self.do_task, task_id)