Made Attribute value queries also work with database objects by overloading the Attribute manager methods in question. Added procpool support for the new serializer functions and cleaned up some things.

contrib/procpools/python_procpool.py

@@ -31,7 +31,8 @@ _return statement, to test it really is asynchronous.
 from twisted.protocols import amp
 from twisted.internet import threads
 from contrib.procpools.ampoule.child import AMPChild
-from src.utils.utils import to_pickle, from_pickle, clean_object_caches, to_str
+from src.utils.dbserialize import to_pickle, from_pickle, do_pickle, do_unpickle
+from src.utils.utils import clean_object_caches, to_str
 from src.utils import logger
 from src import PROC_MODIFIED_OBJS
 
@@ -110,12 +111,11 @@ class PythonProcPoolChild(AMPChild):
                 self.returns.extend(list(args))
             def get_returns(self):
                 lr = len(self.returns)
-                if lr == 0:
-                    return ""
-                elif lr == 1:
-                    return to_pickle(self.returns[0], emptypickle=False) or ""
+                val = lr and (lr == 1 and self.returns[0] or self.returns) or None
+                if val not in (None, [], ()):
+                    return do_pickle(to_pickle(val))
                 else:
-                    return to_pickle(self.returns, emptypickle=False) or ""
+                    return ""
         _return = Ret()
 
 
@@ -123,14 +123,17 @@ class PythonProcPoolChild(AMPChild):
         if environment:
             # load environment
             try:
-                environment = from_pickle(environment)
+                environment = from_pickle(do_unpickle(environment))
                 available_vars.update(environment)
             except Exception:
                 logger.log_trace()
         # try to execute with eval first
         try:
             ret = eval(source, {}, available_vars)
-            ret = _return.get_returns() or to_pickle(ret, emptypickle=False) or ""
+            if ret not in (None, [], ()):
+                ret = _return.get_returns() or do_pickle(to_pickle(ret))
+            else:
+                ret = ""
         except Exception:
             # use exec instead
             exec source in available_vars
@@ -141,7 +144,10 @@ class PythonProcPoolChild(AMPChild):
         objs = objs + list(set([o.location for o in objs if hasattr(o, "location") and o.location]))
         #print "objs:", objs
         #print "to_pickle", to_pickle(objs, emptypickle=False, do_pickle=False)
-        to_recache = to_pickle(objs, emptypickle=False) or ""
+        if objs not in (None, [], ()):
+            to_recache = do_pickle(to_pickle(objs))
+        else:
+            to_recache = ""
         # empty the list without loosing memory reference
         PROC_MODIFIED_OBJS[:] = []
         return {'response': ret,
@@ -250,8 +256,8 @@ def run_async(to_execute, *args, **kwargs):
     # helper converters for callbacks/errbacks
     def convert_return(f):
         def func(ret, *args, **kwargs):
-            rval = ret["response"] and from_pickle(ret["response"])
-            reca = ret["recached"] and from_pickle(ret["recached"])
+            rval = ret["response"] and from_pickle(do_unpickle(ret["response"]))
+            reca = ret["recached"] and from_pickle(do_unpickle(ret["recached"]))
             # recache all indicated objects
             [clean_object_caches(obj) for obj in reca]
             if f: return f(rval, *args, **kwargs)
@@ -283,7 +289,8 @@ def run_async(to_execute, *args, **kwargs):
             # run source code in process pool
             cmdargs = {"_timeout":use_timeout}
             cmdargs["source"] = to_str(to_execute)
-            cmdargs["environment"] = to_pickle(kwargs, emptypickle=False) or ""
+            if kwargs: cmdargs["environment"] = do_pickle(to_pickle(kwargs))
+            else: cmdargs["environment"] = ""
             # defer to process pool
             deferred = _PPOOL.doWork(ExecuteCode, **cmdargs)
         elif callable(to_execute):
@@ -291,7 +298,7 @@ def run_async(to_execute, *args, **kwargs):
             callname = to_execute.__name__
             cmdargs = {"_timeout":use_timeout}
             cmdargs["source"] = "_return(%s(*args,**kwargs))" % callname
-            cmdargs["environment"] = to_pickle({callname:to_execute, "args":args, "kwargs":kwargs})
+            cmdargs["environment"] = do_pickle(to_pickle({callname:to_execute, "args":args, "kwargs":kwargs}))
             deferred = _PPOOL.doWork(ExecuteCode, **cmdargs)
         else:
             raise RuntimeError("'%s' could not be handled by the process pool" % to_execute)

game/evennia.py

@@ -159,6 +159,7 @@ except ObjectDB.DoesNotExist:
 except DatabaseError,e:
     print """
     Your database does not seem to be set up correctly.
+    (error was '%s')
 
     Please run:
 
@@ -171,8 +172,7 @@ except DatabaseError,e:
          python manage.py migrate
 
     When you have a database set up, rerun evennia.py.
-    """
-    print e
+    """ % e
     sys.exit()
 
 # Add this to the environmental variable for the 'twistd' command.

src/objects/manager.py

@@ -5,7 +5,6 @@ try: import cPickle as pickle
 except ImportError: import pickle
 from django.db.models import Q
 from django.conf import settings
-#from django.contrib.auth.models import User
 from django.db.models.fields import exceptions
 from src.typeclasses.managers import TypedObjectManager
 from src.typeclasses.managers import returns_typeclass, returns_typeclass_list

src/typeclasses/managers.py

@@ -7,12 +7,46 @@ from functools import update_wrapper
 from django.db import models
 from src.utils import idmapper
 from src.utils.utils import make_iter
+from src.utils.dbserialize import to_pickle
+
 __all__ = ("AttributeManager", "TypedObjectManager")
 
 # Managers
 
+def _attr_pickled(method):
+    """
+    decorator for safely handling attribute searches
+    - db_value is a pickled field and this is required
+    in order to be able for pickled django objects directly.
+    """
+    def wrapper(self, *args, **kwargs):
+        "wrap all queries searching the db_value field in some way"
+        self.__doc__ = method.__doc__
+        for key in (key for key in kwargs if key.startswith('db_value')):
+            kwargs[key] = to_pickle(kwargs[key])
+        return method(self, *args, **kwargs)
+    return update_wrapper(wrapper, method)
+
 class AttributeManager(models.Manager):
     "Manager for handling Attributes."
+    @_attr_pickled
+    def get(self, *args, **kwargs):
+        return super(AttributeManager, self).get(*args, **kwargs)
+    @_attr_pickled
+    def filter(self,*args, **kwargs):
+        return super(AttributeManager, self).filter(*args, **kwargs)
+    @_attr_pickled
+    def exclude(self,*args, **kwargs):
+        return super(AttributeManager, self).exclude(*args, **kwargs)
+    @_attr_pickled
+    def values(self,*args, **kwargs):
+        return super(AttributeManager, self).values(*args, **kwargs)
+    @_attr_pickled
+    def values_list(self,*args, **kwargs):
+        return super(AttributeManager, self).values_list(*args, **kwargs)
+    @_attr_pickled
+    def exists(self,*args, **kwargs):
+        return super(AttributeManager, self).exists(*args, **kwargs)
 
     def attr_namesearch(self, searchstr, obj, exact_match=True):
         """
@@ -28,6 +62,14 @@ class AttributeManager(models.Manager):
             return self.filter(db_obj=obj).filter(
                 db_key__icontains=searchstr)
 
+    def attr_valuesearch(self, searchstr, obj=None):
+        """
+        Searches for Attributes with a given value on obj
+        """
+        if obj:
+            return self.filter(db_obj=obj, db_value=searchstr)
+        return self.filter(db_value=searchstr)
+
 #
 # helper functions for the TypedObjectManager.
 #

src/utils/dbserialize.py

@@ -19,6 +19,7 @@ be out of sync with the database.
 
 """
 
+from functools import update_wrapper
 from collections import defaultdict, MutableSequence, MutableSet, MutableMapping
 try:
     from cPickle import dumps, loads
@@ -32,7 +33,7 @@ from src.utils import logger
 
 __all__ = ("to_pickle", "from_pickle", "do_pickle", "do_unpickle")
 
-HIGHEST_PROTOCOL = 2
+PICKLE_PROTOCOL = 2
 
 # initialization and helpers
 
@@ -61,10 +62,11 @@ def _init_globals():
 def _save(method):
     "method decorator that saves data to Attribute"
     def save_wrapper(self, *args, **kwargs):
+        self.__doc__ = method.__doc__
         ret = method(self, *args, **kwargs)
         self._save_tree()
         return ret
-    return save_wrapper
+    return update_wrapper(save_wrapper, method)
 
 class _SaverMutable(object):
     """
@@ -311,7 +313,7 @@ def from_pickle(data, db_obj=None):
 
 def do_pickle(data):
     "Perform pickle to string"
-    return to_str(dumps(data, protocol=HIGHEST_PROTOCOL))
+    return to_str(dumps(data, protocol=PICKLE_PROTOCOL))
 
 def do_unpickle(data):
     "Retrieve pickle from pickled string"

src/utils/utils.py

@@ -479,103 +479,6 @@ def delay(delay=2, retval=None, callback=None):
     return d
 
 
-_FROM_MODEL_MAP = None
-_TO_DBOBJ = lambda o: (hasattr(o, "dbobj") and o.dbobj) or o
-_TO_PACKED_DBOBJ = lambda natural_key, dbref: ('__packed_dbobj__', natural_key, dbref)
-_DUMPS = None
-def to_pickle(data, do_pickle=True, emptypickle=True):
-    """
-    Prepares object for being pickled. This will remap database models
-    into an intermediary format, making them easily retrievable later.
-
-    obj - a python object to prepare for pickling
-    do_pickle - return a pickled object
-    emptypickle - allow pickling also a None/empty value (False will be pickled)
-                  This has no effect if do_pickle is False
-
-    Database objects are stored as ('__packed_dbobj__', <natural_key_tuple>, <dbref>)
-    """
-    # prepare globals
-    global _DUMPS, _FROM_MODEL_MAP
-    _DUMPS = lambda data: to_str(pickle.dumps(data, pickle.HIGHEST_PROTOCOL))
-    if not _DUMPS:
-        _DUMPS = lambda data: to_str(pickle.dumps(data, pickle.HIGHEST_PROTOCOL))
-    if not _FROM_MODEL_MAP:
-        _FROM_MODEL_MAP = defaultdict(str)
-        _FROM_MODEL_MAP.update(dict((c.model, c.natural_key()) for c in ContentType.objects.all()))
-
-    def iter_db2id(item):
-        "recursively looping over iterable items, finding dbobjs"
-        dtype = type(item)
-        if dtype in (basestring, int, float):
-            return item
-        elif dtype == tuple:
-            return tuple(iter_db2id(val) for val in item)
-        elif dtype == dict:
-            return dict((key, iter_db2id(val)) for key, val in item.items())
-        elif hasattr(item, '__iter__'):
-            return [iter_db2id(val) for val in item]
-        else:
-            item = _TO_DBOBJ(item)
-            natural_key = _FROM_MODEL_MAP[hasattr(item, "id") and hasattr(item, '__class__') and item.__class__.__name__.lower()]
-            if natural_key:
-                return _TO_PACKED_DBOBJ(natural_key, item.id)
-        return item
-    # do recursive conversion
-    data = iter_db2id(data)
-    if do_pickle and not (not emptypickle and not data and data != False):
-        print "_DUMPS2:", _DUMPS
-        return _DUMPS(data)
-    return data
-
-_TO_MODEL_MAP = None
-_IS_PACKED_DBOBJ = lambda o: type(o)== tuple and len(o)==3 and o[0]=='__packed_dbobj__'
-_TO_TYPECLASS = lambda o: (hasattr(o, 'typeclass') and o.typeclass) or o
-_LOADS = None
-from django.db import transaction
-@transaction.autocommit
-def from_pickle(data, do_pickle=True):
-    """
-    Converts back from a data stream prepared with to_pickle. This will
-    re-acquire database objects stored in the special format.
-
-    obj - an object or a pickle, as indicated by the do_pickle flag
-    do_pickle - actually unpickle the input before continuing
-    """
-    # prepare globals
-    global _LOADS, _TO_MODEL_MAP
-    if not _LOADS:
-        _LOADS = lambda data: pickle.loads(to_str(data))
-    if not _TO_MODEL_MAP:
-        _TO_MODEL_MAP = defaultdict(str)
-        _TO_MODEL_MAP.update(dict((c.natural_key(), c.model_class()) for c in ContentType.objects.all()))
-
-    def iter_id2db(item):
-        "Recreate all objects recursively"
-        dtype = type(item)
-        if dtype in (basestring, int, float):
-            return item
-        elif _IS_PACKED_DBOBJ(item): # this is a tuple and must be done before tuple-check
-            #print item[1], item[2]
-            if item[2]: #Not sure why this could ever be None, but it can
-                return  _TO_TYPECLASS(_TO_MODEL_MAP[item[1]].objects.get(id=item[2]))
-            return None
-        elif dtype == tuple:
-            return tuple(iter_id2db(val) for val in item)
-        elif dtype == dict:
-            return dict((key, iter_id2db(val)) for key, val in item.items())
-        elif hasattr(item, '__iter__'):
-            return [iter_id2db(val) for val in item]
-        return item
-    if do_pickle:
-        data = _LOADS(data)
-    # we have to make sure the database is in a safe state
-    # (this is relevant for multiprocess operation)
-    transaction.commit()
-    # do recursive conversion
-    return iter_id2db(data)
-
-
 _TYPECLASSMODELS = None
 _OBJECTMODELS = None
 def clean_object_caches(obj):
@@ -996,3 +899,4 @@ def format_table(table, extra_space=1):
         ftable.append([str(col[irow]).ljust(max_widths[icol]) + " " * extra_space
                        for icol, col in enumerate(table)])
     return ftable
+