diff --git a/src/objects/migrations/0009_converting_attributes.py b/src/objects/migrations/0009_converting_attributes.py index 4d72409318..9195f7fde7 100644 --- a/src/objects/migrations/0009_converting_attributes.py +++ b/src/objects/migrations/0009_converting_attributes.py @@ -9,7 +9,143 @@ try: except ImportError: import pickle from src.utils.utils import to_str, to_unicode -from src.typeclasses.models import PackedDBobject +from src.typeclasses.models import PackedDBobject,PackedDict,PackedList + +from django.contrib.contenttypes.models import ContentType +CTYPEGET = ContentType.objects.get +GA = object.__getattribute__ +SA = object.__setattr__ +DA = object.__delattr__ + +def to_attr(data): + """ + Convert data to proper attr data format before saving + + We have to make sure to not store database objects raw, since + this will crash the system. Instead we must store their IDs + and make sure to convert back when the attribute is read back + later. + + Due to this it's criticial that we check all iterables + recursively, converting all found database objects to a form + the database can handle. We handle lists, tuples and dicts + (and any nested combination of them) this way, all other + iterables are stored and returned as lists. + + data storage format: + (simple|dbobj|iter, ) + where + simple - a single non-db object, like a string or number + dbobj - a single dbobj + iter - any iterable object - will be looped over recursively + to convert dbobj->id. + + """ + + def iter_db2id(item): + """ + recursively looping through stored iterables, replacing objects with ids. + (Python only builds nested functions once, so there is no overhead for nesting) + """ + dtype = type(item) + if dtype in (basestring, int, float): # check the most common types first, for speed + return item + elif hasattr(item, "id") and hasattr(item, "db_model_name") and hasattr(item, "db_key"): + db_model_name = item.db_model_name + if db_model_name == "typeclass": + db_model_name = GA(item.dbobj, "db_model_name") + return PackedDBobject(item.id, db_model_name, item.db_key) + elif dtype == tuple: + return tuple(iter_db2id(val) for val in item) + elif dtype in (dict, PackedDict): + return dict((key, iter_db2id(val)) for key, val in item.items()) + elif hasattr(item, '__iter__'): + return list(iter_db2id(val) for val in item) + else: + return item + + dtype = type(data) + + if dtype in (basestring, int, float): + return ("simple",data) + elif hasattr(data, "id") and hasattr(data, "db_model_name") and hasattr(data, 'db_key'): + # all django models (objectdb,scriptdb,playerdb,channel,msg,typeclass) + # have the protected property db_model_name hardcoded on themselves for speed. + db_model_name = data.db_model_name + if db_model_name == "typeclass": + # typeclass cannot help us, we want the actual child object model name + db_model_name = GA(data.dbobj, "db_model_name") + return ("dbobj", PackedDBobject(data.id, db_model_name, data.db_key)) + elif hasattr(data, "__iter__"): + return ("iter", iter_db2id(data)) + else: + return ("simple", data) + +def from_attr(attr, datatuple): + """ + Retrieve data from a previously stored attribute. This + is always a dict with keys type and data. + + datatuple comes from the database storage and has + the following format: + (simple|dbobj|iter, ) + where + simple - a single non-db object, like a string. is returned as-is. + dbobj - a single dbobj-id. This id is retrieved back from the database. + iter - an iterable. This is traversed iteratively, converting all found + dbobj-ids back to objects. Also, all lists and dictionaries are + returned as their PackedList/PackedDict counterparts in order to + allow in-place assignment such as obj.db.mylist[3] = val. Mylist + is then a PackedList that saves the data on the fly. + """ + # nested functions + def id2db(data): + """ + Convert db-stored dbref back to object + """ + mclass = CTYPEGET(model=data.db_model).model_class() + try: + return mclass.objects.dbref_search(data.id) + + except AttributeError: + try: + return mclass.objects.get(id=data.id) + except mclass.DoesNotExist: # could happen if object was deleted in the interim. + return None + + def iter_id2db(item): + """ + Recursively looping through stored iterables, replacing ids with actual objects. + We return PackedDict and PackedLists instead of normal lists; this is needed in order for + the user to do dynamic saving of nested in-place, such as obj.db.attrlist[2]=3. What is + stored in the database are however always normal python primitives. + """ + dtype = type(item) + if dtype in (basestring, int, float): # check the most common types first, for speed + return item + elif dtype == PackedDBobject: + return id2db(item) + elif dtype == tuple: + return tuple([iter_id2db(val) for val in item]) + elif dtype in (dict, PackedDict): + return PackedDict(attr, dict(zip([key for key in item.keys()], + [iter_id2db(val) for val in item.values()]))) + elif hasattr(item, '__iter__'): + return PackedList(attr, list(iter_id2db(val) for val in item)) + else: + return item + + typ, data = datatuple + + if typ == 'simple': + # single non-db objects + return data + elif typ == 'dbobj': + # a single stored dbobj + return id2db(data) + elif typ == 'iter': + # all types of iterables + return iter_id2db(data) class Migration(DataMigration): @@ -24,7 +160,8 @@ class Migration(DataMigration): val = ("dbobj", val) else: val = ("simple", val) - attr.value = attr.from_attr(val) + attr.db_value = to_unicode(pickle.dumps(to_str(to_attr(from_attr(attr, val))))) + attr.save() def backwards(self, orm): "Write your backwards methods here." diff --git a/src/players/migrations/0008_converting_attributes.py b/src/players/migrations/0008_converting_attributes.py index 1314bcf6b4..a790d5868d 100644 --- a/src/players/migrations/0008_converting_attributes.py +++ b/src/players/migrations/0008_converting_attributes.py @@ -9,7 +9,145 @@ try: except ImportError: import pickle from src.utils.utils import to_str, to_unicode -from src.typeclasses.models import PackedDBobject +from src.typeclasses.models import PackedDBobject,PackedDict,PackedList +from src.players.models import PlayerAttribute + +from django.contrib.contenttypes.models import ContentType +CTYPEGET = ContentType.objects.get +GA = object.__getattribute__ +SA = object.__setattr__ +DA = object.__delattr__ + +def to_attr(data): + """ + Convert data to proper attr data format before saving + + We have to make sure to not store database objects raw, since + this will crash the system. Instead we must store their IDs + and make sure to convert back when the attribute is read back + later. + + Due to this it's criticial that we check all iterables + recursively, converting all found database objects to a form + the database can handle. We handle lists, tuples and dicts + (and any nested combination of them) this way, all other + iterables are stored and returned as lists. + + data storage format: + (simple|dbobj|iter, ) + where + simple - a single non-db object, like a string or number + dbobj - a single dbobj + iter - any iterable object - will be looped over recursively + to convert dbobj->id. + + """ + + def iter_db2id(item): + """ + recursively looping through stored iterables, replacing objects with ids. + (Python only builds nested functions once, so there is no overhead for nesting) + """ + dtype = type(item) + if dtype in (basestring, int, float): # check the most common types first, for speed + return item + elif hasattr(item, "id") and hasattr(item, "db_model_name") and hasattr(item, "db_key"): + db_model_name = item.db_model_name + if db_model_name == "typeclass": + db_model_name = GA(item.dbobj, "db_model_name") + return PackedDBobject(item.id, db_model_name, item.db_key) + elif dtype == tuple: + return tuple(iter_db2id(val) for val in item) + elif dtype in (dict, PackedDict): + return dict((key, iter_db2id(val)) for key, val in item.items()) + elif hasattr(item, '__iter__'): + return list(iter_db2id(val) for val in item) + else: + return item + + dtype = type(data) + + if dtype in (basestring, int, float): + return ("simple",data) + elif hasattr(data, "id") and hasattr(data, "db_model_name") and hasattr(data, 'db_key'): + # all django models (objectdb,scriptdb,playerdb,channel,msg,typeclass) + # have the protected property db_model_name hardcoded on themselves for speed. + db_model_name = data.db_model_name + if db_model_name == "typeclass": + # typeclass cannot help us, we want the actual child object model name + db_model_name = GA(data.dbobj, "db_model_name") + return ("dbobj", PackedDBobject(data.id, db_model_name, data.db_key)) + elif hasattr(data, "__iter__"): + return ("iter", iter_db2id(data)) + else: + return ("simple", data) + +def from_attr(attr, datatuple): + """ + Retrieve data from a previously stored attribute. This + is always a dict with keys type and data. + + datatuple comes from the database storage and has + the following format: + (simple|dbobj|iter, ) + where + simple - a single non-db object, like a string. is returned as-is. + dbobj - a single dbobj-id. This id is retrieved back from the database. + iter - an iterable. This is traversed iteratively, converting all found + dbobj-ids back to objects. Also, all lists and dictionaries are + returned as their PackedList/PackedDict counterparts in order to + allow in-place assignment such as obj.db.mylist[3] = val. Mylist + is then a PackedList that saves the data on the fly. + """ + # nested functions + def id2db(data): + """ + Convert db-stored dbref back to object + """ + mclass = CTYPEGET(model=data.db_model).model_class() + try: + return mclass.objects.dbref_search(data.id) + + except AttributeError: + try: + return mclass.objects.get(id=data.id) + except mclass.DoesNotExist: # could happen if object was deleted in the interim. + return None + + def iter_id2db(item): + """ + Recursively looping through stored iterables, replacing ids with actual objects. + We return PackedDict and PackedLists instead of normal lists; this is needed in order for + the user to do dynamic saving of nested in-place, such as obj.db.attrlist[2]=3. What is + stored in the database are however always normal python primitives. + """ + dtype = type(item) + if dtype in (basestring, int, float): # check the most common types first, for speed + return item + elif dtype == PackedDBobject: + return id2db(item) + elif dtype == tuple: + return tuple([iter_id2db(val) for val in item]) + elif dtype in (dict, PackedDict): + return PackedDict(attr, dict(zip([key for key in item.keys()], + [iter_id2db(val) for val in item.values()]))) + elif hasattr(item, '__iter__'): + return PackedList(attr, list(iter_id2db(val) for val in item)) + else: + return item + + typ, data = datatuple + + if typ == 'simple': + # single non-db objects + return data + elif typ == 'dbobj': + # a single stored dbobj + return id2db(data) + elif typ == 'iter': + # all types of iterables + return iter_id2db(data) + class Migration(DataMigration): @@ -24,7 +162,8 @@ class Migration(DataMigration): val = ("dbobj", val) else: val = ("simple", val) - attr.value = attr.from_attr(val) + attr.db_value = to_unicode(pickle.dumps(to_str(to_attr(from_attr(attr, val))))) + attr.save() def backwards(self, orm): "Write your backwards methods here." diff --git a/src/scripts/migrations/0005_converting_attributes.py b/src/scripts/migrations/0005_converting_attributes.py index 80acc35324..fd6c22bba0 100644 --- a/src/scripts/migrations/0005_converting_attributes.py +++ b/src/scripts/migrations/0005_converting_attributes.py @@ -9,7 +9,143 @@ try: except ImportError: import pickle from src.utils.utils import to_str, to_unicode -from src.typeclasses.models import PackedDBobject +from src.typeclasses.models import PackedDBobject,PackedDict,PackedList + +from django.contrib.contenttypes.models import ContentType +CTYPEGET = ContentType.objects.get +GA = object.__getattribute__ +SA = object.__setattr__ +DA = object.__delattr__ + +def to_attr(data): + """ + Convert data to proper attr data format before saving + + We have to make sure to not store database objects raw, since + this will crash the system. Instead we must store their IDs + and make sure to convert back when the attribute is read back + later. + + Due to this it's criticial that we check all iterables + recursively, converting all found database objects to a form + the database can handle. We handle lists, tuples and dicts + (and any nested combination of them) this way, all other + iterables are stored and returned as lists. + + data storage format: + (simple|dbobj|iter, ) + where + simple - a single non-db object, like a string or number + dbobj - a single dbobj + iter - any iterable object - will be looped over recursively + to convert dbobj->id. + + """ + + def iter_db2id(item): + """ + recursively looping through stored iterables, replacing objects with ids. + (Python only builds nested functions once, so there is no overhead for nesting) + """ + dtype = type(item) + if dtype in (basestring, int, float): # check the most common types first, for speed + return item + elif hasattr(item, "id") and hasattr(item, "db_model_name") and hasattr(item, "db_key"): + db_model_name = item.db_model_name + if db_model_name == "typeclass": + db_model_name = GA(item.dbobj, "db_model_name") + return PackedDBobject(item.id, db_model_name, item.db_key) + elif dtype == tuple: + return tuple(iter_db2id(val) for val in item) + elif dtype in (dict, PackedDict): + return dict((key, iter_db2id(val)) for key, val in item.items()) + elif hasattr(item, '__iter__'): + return list(iter_db2id(val) for val in item) + else: + return item + + dtype = type(data) + + if dtype in (basestring, int, float): + return ("simple",data) + elif hasattr(data, "id") and hasattr(data, "db_model_name") and hasattr(data, 'db_key'): + # all django models (objectdb,scriptdb,playerdb,channel,msg,typeclass) + # have the protected property db_model_name hardcoded on themselves for speed. + db_model_name = data.db_model_name + if db_model_name == "typeclass": + # typeclass cannot help us, we want the actual child object model name + db_model_name = GA(data.dbobj, "db_model_name") + return ("dbobj", PackedDBobject(data.id, db_model_name, data.db_key)) + elif hasattr(data, "__iter__"): + return ("iter", iter_db2id(data)) + else: + return ("simple", data) + +def from_attr(attr, datatuple): + """ + Retrieve data from a previously stored attribute. This + is always a dict with keys type and data. + + datatuple comes from the database storage and has + the following format: + (simple|dbobj|iter, ) + where + simple - a single non-db object, like a string. is returned as-is. + dbobj - a single dbobj-id. This id is retrieved back from the database. + iter - an iterable. This is traversed iteratively, converting all found + dbobj-ids back to objects. Also, all lists and dictionaries are + returned as their PackedList/PackedDict counterparts in order to + allow in-place assignment such as obj.db.mylist[3] = val. Mylist + is then a PackedList that saves the data on the fly. + """ + # nested functions + def id2db(data): + """ + Convert db-stored dbref back to object + """ + mclass = CTYPEGET(model=data.db_model).model_class() + try: + return mclass.objects.dbref_search(data.id) + + except AttributeError: + try: + return mclass.objects.get(id=data.id) + except mclass.DoesNotExist: # could happen if object was deleted in the interim. + return None + + def iter_id2db(item): + """ + Recursively looping through stored iterables, replacing ids with actual objects. + We return PackedDict and PackedLists instead of normal lists; this is needed in order for + the user to do dynamic saving of nested in-place, such as obj.db.attrlist[2]=3. What is + stored in the database are however always normal python primitives. + """ + dtype = type(item) + if dtype in (basestring, int, float): # check the most common types first, for speed + return item + elif dtype == PackedDBobject: + return id2db(item) + elif dtype == tuple: + return tuple([iter_id2db(val) for val in item]) + elif dtype in (dict, PackedDict): + return PackedDict(attr, dict(zip([key for key in item.keys()], + [iter_id2db(val) for val in item.values()]))) + elif hasattr(item, '__iter__'): + return PackedList(attr, list(iter_id2db(val) for val in item)) + else: + return item + + typ, data = datatuple + + if typ == 'simple': + # single non-db objects + return data + elif typ == 'dbobj': + # a single stored dbobj + return id2db(data) + elif typ == 'iter': + # all types of iterables + return iter_id2db(data) class Migration(DataMigration): @@ -24,7 +160,8 @@ class Migration(DataMigration): val = ("dbobj", val) else: val = ("simple", val) - attr.value = attr.from_attr(val) + attr.db_value = to_unicode(pickle.dumps(to_str(to_attr(from_attr(attr, val))))) + attr.save() def backwards(self, orm): "Write your backwards methods here."