From 80a6745a1ed05c14145a35acb44e6a8cb412584b Mon Sep 17 00:00:00 2001 From: Griatch Date: Sun, 14 Apr 2013 16:36:44 +0200 Subject: [PATCH] Made Attribute value queries also work with database objects by overloading the Attribute manager methods in question. Added procpool support for the new serializer functions and cleaned up some things. --- contrib/procpools/python_procpool.py | 33 ++++++---- game/evennia.py | 4 +- src/objects/manager.py | 1 - src/typeclasses/managers.py | 42 ++++++++++++ src/utils/dbserialize.py | 8 ++- src/utils/utils.py | 98 +--------------------------- 6 files changed, 70 insertions(+), 116 deletions(-) diff --git a/contrib/procpools/python_procpool.py b/contrib/procpools/python_procpool.py index 47676fed45..04ab0dac95 100644 --- a/contrib/procpools/python_procpool.py +++ b/contrib/procpools/python_procpool.py @@ -31,7 +31,8 @@ _return statement, to test it really is asynchronous. from twisted.protocols import amp from twisted.internet import threads from contrib.procpools.ampoule.child import AMPChild -from src.utils.utils import to_pickle, from_pickle, clean_object_caches, to_str +from src.utils.dbserialize import to_pickle, from_pickle, do_pickle, do_unpickle +from src.utils.utils import clean_object_caches, to_str from src.utils import logger from src import PROC_MODIFIED_OBJS @@ -110,12 +111,11 @@ class PythonProcPoolChild(AMPChild): self.returns.extend(list(args)) def get_returns(self): lr = len(self.returns) - if lr == 0: - return "" - elif lr == 1: - return to_pickle(self.returns[0], emptypickle=False) or "" + val = lr and (lr == 1 and self.returns[0] or self.returns) or None + if val not in (None, [], ()): + return do_pickle(to_pickle(val)) else: - return to_pickle(self.returns, emptypickle=False) or "" + return "" _return = Ret() @@ -123,14 +123,17 @@ class PythonProcPoolChild(AMPChild): if environment: # load environment try: - environment = from_pickle(environment) + environment = from_pickle(do_unpickle(environment)) available_vars.update(environment) except Exception: logger.log_trace() # try to execute with eval first try: ret = eval(source, {}, available_vars) - ret = _return.get_returns() or to_pickle(ret, emptypickle=False) or "" + if ret not in (None, [], ()): + ret = _return.get_returns() or do_pickle(to_pickle(ret)) + else: + ret = "" except Exception: # use exec instead exec source in available_vars @@ -141,7 +144,10 @@ class PythonProcPoolChild(AMPChild): objs = objs + list(set([o.location for o in objs if hasattr(o, "location") and o.location])) #print "objs:", objs #print "to_pickle", to_pickle(objs, emptypickle=False, do_pickle=False) - to_recache = to_pickle(objs, emptypickle=False) or "" + if objs not in (None, [], ()): + to_recache = do_pickle(to_pickle(objs)) + else: + to_recache = "" # empty the list without loosing memory reference PROC_MODIFIED_OBJS[:] = [] return {'response': ret, @@ -250,8 +256,8 @@ def run_async(to_execute, *args, **kwargs): # helper converters for callbacks/errbacks def convert_return(f): def func(ret, *args, **kwargs): - rval = ret["response"] and from_pickle(ret["response"]) - reca = ret["recached"] and from_pickle(ret["recached"]) + rval = ret["response"] and from_pickle(do_unpickle(ret["response"])) + reca = ret["recached"] and from_pickle(do_unpickle(ret["recached"])) # recache all indicated objects [clean_object_caches(obj) for obj in reca] if f: return f(rval, *args, **kwargs) @@ -283,7 +289,8 @@ def run_async(to_execute, *args, **kwargs): # run source code in process pool cmdargs = {"_timeout":use_timeout} cmdargs["source"] = to_str(to_execute) - cmdargs["environment"] = to_pickle(kwargs, emptypickle=False) or "" + if kwargs: cmdargs["environment"] = do_pickle(to_pickle(kwargs)) + else: cmdargs["environment"] = "" # defer to process pool deferred = _PPOOL.doWork(ExecuteCode, **cmdargs) elif callable(to_execute): @@ -291,7 +298,7 @@ def run_async(to_execute, *args, **kwargs): callname = to_execute.__name__ cmdargs = {"_timeout":use_timeout} cmdargs["source"] = "_return(%s(*args,**kwargs))" % callname - cmdargs["environment"] = to_pickle({callname:to_execute, "args":args, "kwargs":kwargs}) + cmdargs["environment"] = do_pickle(to_pickle({callname:to_execute, "args":args, "kwargs":kwargs})) deferred = _PPOOL.doWork(ExecuteCode, **cmdargs) else: raise RuntimeError("'%s' could not be handled by the process pool" % to_execute) diff --git a/game/evennia.py b/game/evennia.py index fbaf7e5c26..54d6282317 100755 --- a/game/evennia.py +++ b/game/evennia.py @@ -159,6 +159,7 @@ except ObjectDB.DoesNotExist: except DatabaseError,e: print """ Your database does not seem to be set up correctly. + (error was '%s') Please run: @@ -171,8 +172,7 @@ except DatabaseError,e: python manage.py migrate When you have a database set up, rerun evennia.py. - """ - print e + """ % e sys.exit() # Add this to the environmental variable for the 'twistd' command. diff --git a/src/objects/manager.py b/src/objects/manager.py index ec7f107ab5..7117527dac 100644 --- a/src/objects/manager.py +++ b/src/objects/manager.py @@ -5,7 +5,6 @@ try: import cPickle as pickle except ImportError: import pickle from django.db.models import Q from django.conf import settings -#from django.contrib.auth.models import User from django.db.models.fields import exceptions from src.typeclasses.managers import TypedObjectManager from src.typeclasses.managers import returns_typeclass, returns_typeclass_list diff --git a/src/typeclasses/managers.py b/src/typeclasses/managers.py index 9b61a0d0f7..e3630efe47 100644 --- a/src/typeclasses/managers.py +++ b/src/typeclasses/managers.py @@ -7,12 +7,46 @@ from functools import update_wrapper from django.db import models from src.utils import idmapper from src.utils.utils import make_iter +from src.utils.dbserialize import to_pickle + __all__ = ("AttributeManager", "TypedObjectManager") # Managers +def _attr_pickled(method): + """ + decorator for safely handling attribute searches + - db_value is a pickled field and this is required + in order to be able for pickled django objects directly. + """ + def wrapper(self, *args, **kwargs): + "wrap all queries searching the db_value field in some way" + self.__doc__ = method.__doc__ + for key in (key for key in kwargs if key.startswith('db_value')): + kwargs[key] = to_pickle(kwargs[key]) + return method(self, *args, **kwargs) + return update_wrapper(wrapper, method) + class AttributeManager(models.Manager): "Manager for handling Attributes." + @_attr_pickled + def get(self, *args, **kwargs): + return super(AttributeManager, self).get(*args, **kwargs) + @_attr_pickled + def filter(self,*args, **kwargs): + return super(AttributeManager, self).filter(*args, **kwargs) + @_attr_pickled + def exclude(self,*args, **kwargs): + return super(AttributeManager, self).exclude(*args, **kwargs) + @_attr_pickled + def values(self,*args, **kwargs): + return super(AttributeManager, self).values(*args, **kwargs) + @_attr_pickled + def values_list(self,*args, **kwargs): + return super(AttributeManager, self).values_list(*args, **kwargs) + @_attr_pickled + def exists(self,*args, **kwargs): + return super(AttributeManager, self).exists(*args, **kwargs) def attr_namesearch(self, searchstr, obj, exact_match=True): """ @@ -28,6 +62,14 @@ class AttributeManager(models.Manager): return self.filter(db_obj=obj).filter( db_key__icontains=searchstr) + def attr_valuesearch(self, searchstr, obj=None): + """ + Searches for Attributes with a given value on obj + """ + if obj: + return self.filter(db_obj=obj, db_value=searchstr) + return self.filter(db_value=searchstr) + # # helper functions for the TypedObjectManager. # diff --git a/src/utils/dbserialize.py b/src/utils/dbserialize.py index 6ccc4e5a5a..156a076468 100644 --- a/src/utils/dbserialize.py +++ b/src/utils/dbserialize.py @@ -19,6 +19,7 @@ be out of sync with the database. """ +from functools import update_wrapper from collections import defaultdict, MutableSequence, MutableSet, MutableMapping try: from cPickle import dumps, loads @@ -32,7 +33,7 @@ from src.utils import logger __all__ = ("to_pickle", "from_pickle", "do_pickle", "do_unpickle") -HIGHEST_PROTOCOL = 2 +PICKLE_PROTOCOL = 2 # initialization and helpers @@ -61,10 +62,11 @@ def _init_globals(): def _save(method): "method decorator that saves data to Attribute" def save_wrapper(self, *args, **kwargs): + self.__doc__ = method.__doc__ ret = method(self, *args, **kwargs) self._save_tree() return ret - return save_wrapper + return update_wrapper(save_wrapper, method) class _SaverMutable(object): """ @@ -311,7 +313,7 @@ def from_pickle(data, db_obj=None): def do_pickle(data): "Perform pickle to string" - return to_str(dumps(data, protocol=HIGHEST_PROTOCOL)) + return to_str(dumps(data, protocol=PICKLE_PROTOCOL)) def do_unpickle(data): "Retrieve pickle from pickled string" diff --git a/src/utils/utils.py b/src/utils/utils.py index ccaa00d78c..7579774230 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -479,103 +479,6 @@ def delay(delay=2, retval=None, callback=None): return d -_FROM_MODEL_MAP = None -_TO_DBOBJ = lambda o: (hasattr(o, "dbobj") and o.dbobj) or o -_TO_PACKED_DBOBJ = lambda natural_key, dbref: ('__packed_dbobj__', natural_key, dbref) -_DUMPS = None -def to_pickle(data, do_pickle=True, emptypickle=True): - """ - Prepares object for being pickled. This will remap database models - into an intermediary format, making them easily retrievable later. - - obj - a python object to prepare for pickling - do_pickle - return a pickled object - emptypickle - allow pickling also a None/empty value (False will be pickled) - This has no effect if do_pickle is False - - Database objects are stored as ('__packed_dbobj__', , ) - """ - # prepare globals - global _DUMPS, _FROM_MODEL_MAP - _DUMPS = lambda data: to_str(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) - if not _DUMPS: - _DUMPS = lambda data: to_str(pickle.dumps(data, pickle.HIGHEST_PROTOCOL)) - if not _FROM_MODEL_MAP: - _FROM_MODEL_MAP = defaultdict(str) - _FROM_MODEL_MAP.update(dict((c.model, c.natural_key()) for c in ContentType.objects.all())) - - def iter_db2id(item): - "recursively looping over iterable items, finding dbobjs" - dtype = type(item) - if dtype in (basestring, int, float): - return item - elif dtype == tuple: - return tuple(iter_db2id(val) for val in item) - elif dtype == dict: - return dict((key, iter_db2id(val)) for key, val in item.items()) - elif hasattr(item, '__iter__'): - return [iter_db2id(val) for val in item] - else: - item = _TO_DBOBJ(item) - natural_key = _FROM_MODEL_MAP[hasattr(item, "id") and hasattr(item, '__class__') and item.__class__.__name__.lower()] - if natural_key: - return _TO_PACKED_DBOBJ(natural_key, item.id) - return item - # do recursive conversion - data = iter_db2id(data) - if do_pickle and not (not emptypickle and not data and data != False): - print "_DUMPS2:", _DUMPS - return _DUMPS(data) - return data - -_TO_MODEL_MAP = None -_IS_PACKED_DBOBJ = lambda o: type(o)== tuple and len(o)==3 and o[0]=='__packed_dbobj__' -_TO_TYPECLASS = lambda o: (hasattr(o, 'typeclass') and o.typeclass) or o -_LOADS = None -from django.db import transaction -@transaction.autocommit -def from_pickle(data, do_pickle=True): - """ - Converts back from a data stream prepared with to_pickle. This will - re-acquire database objects stored in the special format. - - obj - an object or a pickle, as indicated by the do_pickle flag - do_pickle - actually unpickle the input before continuing - """ - # prepare globals - global _LOADS, _TO_MODEL_MAP - if not _LOADS: - _LOADS = lambda data: pickle.loads(to_str(data)) - if not _TO_MODEL_MAP: - _TO_MODEL_MAP = defaultdict(str) - _TO_MODEL_MAP.update(dict((c.natural_key(), c.model_class()) for c in ContentType.objects.all())) - - def iter_id2db(item): - "Recreate all objects recursively" - dtype = type(item) - if dtype in (basestring, int, float): - return item - elif _IS_PACKED_DBOBJ(item): # this is a tuple and must be done before tuple-check - #print item[1], item[2] - if item[2]: #Not sure why this could ever be None, but it can - return _TO_TYPECLASS(_TO_MODEL_MAP[item[1]].objects.get(id=item[2])) - return None - elif dtype == tuple: - return tuple(iter_id2db(val) for val in item) - elif dtype == dict: - return dict((key, iter_id2db(val)) for key, val in item.items()) - elif hasattr(item, '__iter__'): - return [iter_id2db(val) for val in item] - return item - if do_pickle: - data = _LOADS(data) - # we have to make sure the database is in a safe state - # (this is relevant for multiprocess operation) - transaction.commit() - # do recursive conversion - return iter_id2db(data) - - _TYPECLASSMODELS = None _OBJECTMODELS = None def clean_object_caches(obj): @@ -996,3 +899,4 @@ def format_table(table, extra_space=1): ftable.append([str(col[irow]).ljust(max_widths[icol]) + " " * extra_space for icol, col in enumerate(table)]) return ftable +