From b6383ddab92acbc32b7cbfbe199504f33f133fe6 Mon Sep 17 00:00:00 2001 From: Griatch Date: Wed, 29 May 2013 18:47:51 +0200 Subject: [PATCH] Moved attr_cache to new caching system, activated all attribute updating signals. --- src/objects/models.py | 7 +++ src/players/models.py | 6 ++ src/scripts/models.py | 6 ++ src/server/caches.py | 115 +++++++++++++++++++------------------ src/typeclasses/models.py | 18 +++--- src/utils/idmapper/base.py | 9 ++- 6 files changed, 95 insertions(+), 66 deletions(-) diff --git a/src/objects/models.py b/src/objects/models.py index a063b256ec..4406902428 100644 --- a/src/objects/models.py +++ b/src/objects/models.py @@ -17,11 +17,13 @@ transparently through the decorating TypeClass. import traceback from django.db import models from django.conf import settings +from django.db.models.signals import post_init, pre_delete from src.utils.idmapper.models import SharedMemoryModel from src.typeclasses.models import Attribute, TypedObject, TypeNick, TypeNickHandler from src.server.caches import get_field_cache, set_field_cache, del_field_cache from src.server.caches import get_prop_cache, set_prop_cache, del_prop_cache +from src.server.caches import attr_post_init, attr_pre_delete from src.typeclasses.typeclass import TypeClass from src.players.models import PlayerNick from src.objects.manager import ObjectManager @@ -53,6 +55,7 @@ _HERE = _("here") # #------------------------------------------------------------ + class ObjAttribute(Attribute): "Attributes for ObjectDB objects." db_obj = models.ForeignKey("ObjectDB") @@ -62,6 +65,10 @@ class ObjAttribute(Attribute): verbose_name = "Object Attribute" verbose_name_plural = "Object Attributes" +# attach the cache handlers for attribute lookup +post_init.connect(attr_post_init, sender=ObjAttribute, dispatch_uid="objattrcache") +pre_delete.connect(attr_pre_delete, sender=ObjAttribute, dispatch_uid="objattrcache") + #------------------------------------------------------------ # # Alias diff --git a/src/players/models.py b/src/players/models.py index 278ab00a14..37e639af18 100644 --- a/src/players/models.py +++ b/src/players/models.py @@ -27,9 +27,12 @@ from django.conf import settings from django.db import models from django.contrib.auth.models import User from django.utils.encoding import smart_str +from django.db.models.signals import post_init, pre_delete from src.server.caches import get_field_cache, set_field_cache, del_field_cache from src.server.caches import get_prop_cache, set_prop_cache, del_prop_cache +from src.server.caches import attr_post_init, attr_pre_delete + from src.players import manager from src.scripts.models import ScriptDB from src.typeclasses.models import Attribute, TypedObject, TypeNick, TypeNickHandler @@ -74,6 +77,9 @@ class PlayerAttribute(Attribute): "Define Django meta options" verbose_name = "Player Attribute" +post_init.connect(attr_post_init, sender=PlayerAttribute, dispatch_uid="playerattrcache") +pre_delete.connect(attr_pre_delete, sender=PlayerAttribute, dispatch_uid="playerattrcache") + #------------------------------------------------------------ # # Player Nicks diff --git a/src/scripts/models.py b/src/scripts/models.py index 538866bca9..ae79794a9a 100644 --- a/src/scripts/models.py +++ b/src/scripts/models.py @@ -26,6 +26,9 @@ Common examples of uses of Scripts: """ from django.conf import settings from django.db import models +from django.db.models.signals import post_init, pre_delete + +from src.server.caches import attr_post_init, attr_pre_delete from src.typeclasses.models import Attribute, TypedObject from django.contrib.contenttypes.models import ContentType from src.scripts.manager import ScriptManager @@ -47,6 +50,9 @@ class ScriptAttribute(Attribute): verbose_name = "Script Attribute" verbose_name_plural = "Script Attributes" +# attach cache handlers for attribute lookup +post_init.connect(attr_post_init, sender=ScriptAttribute, dispatch_uid="scriptattrcache") +pre_delete.connect(attr_pre_delete, sender=ScriptAttribute, dispatch_uid="scriptattrcache") #------------------------------------------------------------ # diff --git a/src/server/caches.py b/src/server/caches.py index d5ad57d6b5..769cbc28b1 100644 --- a/src/server/caches.py +++ b/src/server/caches.py @@ -2,6 +2,8 @@ Central caching module. """ +from django.core.cache import get_cache +#from django.db.models.signals import pre_save, pre_delete, post_init from src.server.models import ServerConfig from src.utils.utils import uses_database, to_str @@ -9,13 +11,27 @@ _GA = object.__getattribute__ _SA = object.__setattr__ _DA = object.__delattr__ +# +# Open handles to the caches +# + +_FIELD_CACHE = get_cache("field_cache") +_ATTR_CACHE = get_cache("attr_cache") + +# make sure caches are empty at startup +_FIELD_CACHE.clear() +_ATTR_CACHE.clear() + +# +# Cache key hash generation +# + if uses_database("mysql") and ServerConfig.objects.get_mysql_db_version() < '5.6.4': # mysql <5.6.4 don't support millisecond precision _DATESTRING = "%Y:%m:%d-%H:%M:%S:000000" else: _DATESTRING = "%Y:%m:%d-%H:%M:%S:%f" - def hashid(obj, suffix=""): """ Returns a per-class unique that combines the object's @@ -49,36 +65,28 @@ def hashid(obj, suffix=""): return to_str(hid) -# signal handlers +# +# Cache callback handlers +# -from django.core.cache import get_cache -#from django.db.models.signals import pre_save, pre_delete, post_init +# Field cache - makes sure to cache all database fields when +# they are saved, no matter from where. -# field cache - -_FIELD_CACHE = get_cache("field_cache") -if not _FIELD_CACHE: - raise RuntimeError("settings.CACHE does not contain a 'field_cache' entry!") - -# callback before saving an object - -def field_pre_save(sender, **kwargs): +# callback to pre_save signal (connected in src.server.server) +def field_pre_save(sender, instance=None, update_fields=None, raw=False, **kwargs): """ Called at the beginning of the save operation. The save method must be called with the update_fields keyword in order to """ - global _FIELD_CACHE - - if kwargs.pop("raw", False): return - instance = kwargs.pop("instance") - fields = kwargs.pop("update_fields", None) - if fields: + if raw: + return + if update_fields: # this is a list of strings at this point. We want field objects - fields = (instance._meta.get_field_by_name(field)[0] for field in fields) + update_fields = (instance._meta.get_field_by_name(field)[0] for field in update_fields) else: # meta.fields are already field objects - fields = instance._meta.fields - for field in fields: + update_fields = instance._meta.fields + for field in update_fields: fieldname = field.name new_value = field.value_from_object(instance) handlername = "_%s_handler" % fieldname @@ -98,39 +106,34 @@ def field_pre_save(sender, **kwargs): # update cache _FIELD_CACHE.set(hid, new_value) -# goes into server: -#pre_save.connect(field_pre_save, dispatch_uid="fieldcache") +# Attr cache - caching the attribute objects related to a given object to +# avoid lookups more than necessary (this makes Attributes en par in speed +# to any property). + +# connected to post_init signal (connected in respective Attribute model) +def attr_post_init(sender, instance=None, **kwargs): + "Called when attribute is created or retrieved in connection with obj." + #print "attr_post_init:", instance, instance.db_obj, instance.db_key + hid = hashid(_GA(instance, "db_obj"), "-%s" % _GA(instance, "db_key")) + if hid: + _ATTR_CACHE.set(hid, sender) +# connected to pre_delete signal (connected in respective Attribute model) +def attr_pre_delete(sender, instance=None, **kwargs): + "Called when attribute is deleted (del_attribute)" + #print "attr_pre_delete:", instance, instance.db_obj, instance.db_key + hid = hashid(_GA(instance, "db_obj"), "-%s" % _GA(instance, "db_key")) + if hid: + #print "attr_pre_delete:", _GA(instance, "db_key") + _ATTR_CACHE.delete(hid) +# access method +def get_attr_cache(obj, attrname): + "Called by get_attribute" + hid = hashid(obj, "-%s" % attrname) + _ATTR_CACHE.delete(hid) + return hid and _ATTR_CACHE.get(hid, None) or None + + -## attr cache - caching the attribute objects related to a given object to -## avoid lookups more than necessary (this makes attributes en par in speed -## to any property). The signal is triggered by the Attribute itself when it -## is created or deleted (it holds a reference to the object) -# -#_ATTR_CACHE = get_cache("attr_cache") -#if not _ATTR_CACHE: -# raise RuntimeError("settings.CACHE does not contain an 'attr_cache' entry!") -# -#def attr_post_init(sender, **kwargs): -# "Called when attribute is created or retrieved in connection with obj." -# hid = hashid(sender.db_obj, "-%s" % sender.db_key) -# _ATTR_CACHE.set(hid, sender) -#def attr_pre_delete(sender, **kwargs): -# "Called when attribute is deleted (del_attribute)" -# hid = hashid(sender.db_obj, "-%s" % sender.db_key) -# _ATTR_CACHE.delete(hid) -# -### goes into server: -#from src.objects.models import ObjAttribute -#from src.scripts.models import ScriptAttribute -#from src.players.models import PlayerAttribute -#post_init.connect(attr_post_init, sender=ObjAttribute, dispatch_uid="objattrcache") -#post_init.connect(attr_post_init, sender=ScriptAttribute, dispatch_uid="scriptattrcache") -#post_init.connect(attr_post_init, sender=PlayerAttribute, dispatch_uid="playerattrcache") -#pre_delete.connect(attr_pre_delete, sender=ObjAttribute, dispatch_uid="objattrcache") -#pre_delete.connect(attr_pre_delete, sender=ScriptAttribute, dispatch_uid="scriptattrcache") -#pre_delete.connect(attr_pre_delete, sender=PlayerAttribute, dispatch_uid="playerattrcache") -# -# ## property cache - this doubles as a central cache and as a way ## to trigger oob on such changes. # @@ -456,8 +459,8 @@ def del_prop_cache(obj, name): pass def flush_prop_cache(obj=None): pass -def get_attr_cache(obj, attrname): - return None +#def get_attr_cache(obj, attrname): +# return None def set_attr_cache(obj, attrname, attrobj): pass def del_attr_cache(obj, attrname): diff --git a/src/typeclasses/models.py b/src/typeclasses/models.py index 745bc98b01..79f1da70a8 100644 --- a/src/typeclasses/models.py +++ b/src/typeclasses/models.py @@ -103,7 +103,7 @@ class Attribute(SharedMemoryModel): # Attribute Database Model setup # # - # These databse fields are all set using their corresponding properties, + # These database fields are all set using their corresponding properties, # named same as the field, but withtout the db_* prefix. db_key = models.CharField('key', max_length=255, db_index=True) @@ -933,10 +933,11 @@ class TypedObject(SharedMemoryModel): if not get_attr_cache(self, attribute_name): attrib_obj = _GA(self, "_attribute_class").objects.filter( db_obj=self, db_key__iexact=attribute_name) - if attrib_obj: - set_attr_cache(self, attribute_name, attrib_obj[0]) - else: + if not attrib_obj: return False + #set_attr_cache(self, attribute_name, attrib_obj[0]) + #else: + # return False return True def set_attribute(self, attribute_name, new_value=None, lockstring=""): @@ -953,6 +954,7 @@ class TypedObject(SharedMemoryModel): types checked by secureattr are 'attrread','attredit','attrcreate'. """ attrib_obj = get_attr_cache(self, attribute_name) + print "set_attribute:", attribute_name, attrib_obj if not attrib_obj: attrclass = _GA(self, "_attribute_class") # check if attribute already exists. @@ -975,7 +977,7 @@ class TypedObject(SharedMemoryModel): flush_attr_cache(self) self.delete() raise IntegrityError("Attribute could not be saved - object %s was deleted from database." % self.key) - set_attr_cache(self, attribute_name, attrib_obj) + #set_attr_cache(self, attribute_name, attrib_obj) def get_attribute_obj(self, attribute_name, default=None): """ @@ -987,7 +989,7 @@ class TypedObject(SharedMemoryModel): db_obj=self, db_key__iexact=attribute_name) if not attrib_obj: return default - set_attr_cache(self, attribute_name, attrib_obj[0]) #query is first evaluated here + #set_attr_cache(self, attribute_name, attrib_obj[0]) #query is first evaluated here return attrib_obj[0] return attrib_obj @@ -1006,7 +1008,7 @@ class TypedObject(SharedMemoryModel): db_obj=self, db_key__iexact=attribute_name) if not attrib_obj: return default - set_attr_cache(self, attribute_name, attrib_obj[0]) #query is first evaluated here + #set_attr_cache(self, attribute_name, attrib_obj[0]) #query is first evaluated here return attrib_obj[0].value return attrib_obj.value @@ -1023,7 +1025,7 @@ class TypedObject(SharedMemoryModel): db_obj=self, db_key__iexact=attribute_name) if not attrib_obj: raise AttributeError - set_attr_cache(self, attribute_name, attrib_obj[0]) #query is first evaluated here + #set_attr_cache(self, attribute_name, attrib_obj[0]) #query is first evaluated here return attrib_obj[0].value return attrib_obj.value diff --git a/src/utils/idmapper/base.py b/src/utils/idmapper/base.py index b50598c5a0..6563b220af 100755 --- a/src/utils/idmapper/base.py +++ b/src/utils/idmapper/base.py @@ -91,7 +91,12 @@ class SharedMemoryModelBase(ModelBase): super(SharedMemoryModelBase, cls)._prepare() def __init__(cls, *args, **kwargs): - "Takes field names db_* and creates property wrappers named without the db_ prefix. So db_key -> key" + """ + Takes field names db_* and creates property wrappers named without the db_ prefix. So db_key -> key + This wrapper happens on the class level, so there is no overhead when creating objects. If a class + already has a wrapper of the given name, the automatic creation is skipped. Note: Remember to + document this auto-wrapping in the class header, this could seem very much like magic to the user otherwise. + """ super(SharedMemoryModelBase, cls).__init__(*args, **kwargs) def create_wrapper(cls, fieldname, wrappername): "Helper method to create property wrappers with unique names (must be in separate call)" @@ -114,7 +119,7 @@ class SharedMemoryModelBase(ModelBase): wrappername = fieldname == "id" and "dbref" or fieldname.replace("db_", "") if not hasattr(cls, wrappername): # make sure not to overload manually created wrappers on the model - print "wrapping %s -> %s" % (fieldname, wrappername) + #print "wrapping %s -> %s" % (fieldname, wrappername) create_wrapper(cls, fieldname, wrappername) class SharedMemoryModel(Model):