Andreas/evennia
1
0
Fork 0
mirror of https://github.com/evennia/evennia.git synced 2026-03-24 08:46:31 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Finished with a working implementation of the new Attribute mechanism using PickledObjectField. First step of migration is finished, but not converting any old attribute data yet.

Browse source
This commit is contained in:
Griatch 2013-04-13 18:13:15 +02:00
parent 75341ade6f
commit 512d7748d1
5 changed files with 399 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

112
src/objects/migrations/0018_add_picklefield.py Normal file
View file

@ -0,0 +1,112 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'ObjAttribute.db_value2'
db.add_column('objects_objattribute', 'db_value2',
self.gf('src.utils.picklefield.PickledObjectField')(null=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'ObjAttribute.db_value2'
db.delete_column('objects_objattribute', 'db_value2')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'objects.alias': {
'Meta': {'object_name': 'Alias'},
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_obj': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['objects.ObjectDB']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'objects.objattribute': {
'Meta': {'object_name': 'ObjAttribute'},
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_obj': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['objects.ObjectDB']"}),
'db_value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'db_value2': ('src.utils.picklefield.PickledObjectField', [], {'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'objects.objectdb': {
'Meta': {'object_name': 'ObjectDB'},
'db_cmdset_storage': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_destination': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'destinations_set'", 'null': 'True', 'to': "orm['objects.ObjectDB']"}),
'db_home': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'homes_set'", 'null': 'True', 'to': "orm['objects.ObjectDB']"}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_location': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'locations_set'", 'null': 'True', 'to': "orm['objects.ObjectDB']"}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_permissions': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'db_player': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['players.PlayerDB']", 'null': 'True', 'blank': 'True'}),
'db_sessid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
'db_typeclass_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'objects.objectnick': {
'Meta': {'unique_together': "(('db_nick', 'db_type', 'db_obj'),)", 'object_name': 'ObjectNick'},
'db_nick': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_obj': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['objects.ObjectDB']"}),
'db_real': ('django.db.models.fields.TextField', [], {}),
'db_type': ('django.db.models.fields.CharField', [], {'default': "'inputline'", 'max_length': '16', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'players.playerdb': {
'Meta': {'object_name': 'PlayerDB'},
'db_cmdset_storage': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_is_connected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_permissions': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'db_typeclass_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
}
}
complete_apps = ['objects']

91
src/players/migrations/0017_add_picklefield.py Normal file
View file

@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'PlayerAttribute.db_value2'
db.add_column('players_playerattribute', 'db_value2',
self.gf('src.utils.picklefield.PickledObjectField')(null=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'PlayerAttribute.db_value2'
db.delete_column('players_playerattribute', 'db_value2')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'players.playerattribute': {
'Meta': {'object_name': 'PlayerAttribute'},
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_obj': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['players.PlayerDB']"}),
'db_value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'db_value2': ('src.utils.picklefield.PickledObjectField', [], {'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'players.playerdb': {
'Meta': {'object_name': 'PlayerDB'},
'db_cmdset_storage': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_is_connected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_permissions': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'db_typeclass_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
},
'players.playernick': {
'Meta': {'unique_together': "(('db_nick', 'db_type', 'db_obj'),)", 'object_name': 'PlayerNick'},
'db_nick': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_obj': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['players.PlayerDB']"}),
'db_real': ('django.db.models.fields.TextField', [], {}),
'db_type': ('django.db.models.fields.CharField', [], {'default': "'inputline'", 'max_length': '16', 'null': 'True', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
}
}
complete_apps = ['players']

114
src/scripts/migrations/0010_add_picklefield.py Normal file
View file

@ -0,0 +1,114 @@
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'ScriptAttribute.db_value2'
db.add_column('scripts_scriptattribute', 'db_value2',
self.gf('src.utils.picklefield.PickledObjectField')(null=True),
keep_default=False)
def backwards(self, orm):
# Deleting field 'ScriptAttribute.db_value2'
db.delete_column('scripts_scriptattribute', 'db_value2')
models = {
'auth.group': {
'Meta': {'object_name': 'Group'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
},
'auth.permission': {
'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
},
'auth.user': {
'Meta': {'object_name': 'User'},
'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
},
'contenttypes.contenttype': {
'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
},
'objects.objectdb': {
'Meta': {'object_name': 'ObjectDB'},
'db_cmdset_storage': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}),
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_destination': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'destinations_set'", 'null': 'True', 'to': "orm['objects.ObjectDB']"}),
'db_home': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'homes_set'", 'null': 'True', 'to': "orm['objects.ObjectDB']"}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_location': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'locations_set'", 'null': 'True', 'to': "orm['objects.ObjectDB']"}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_permissions': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'db_player': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['players.PlayerDB']", 'null': 'True', 'blank': 'True'}),
'db_sessid': ('django.db.models.fields.IntegerField', [], {'null': 'True'}),
'db_typeclass_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'players.playerdb': {
'Meta': {'object_name': 'PlayerDB'},
'db_cmdset_storage': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_is_connected': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_permissions': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'db_typeclass_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
},
'scripts.scriptattribute': {
'Meta': {'object_name': 'ScriptAttribute'},
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_obj': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['scripts.ScriptDB']"}),
'db_value': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
'db_value2': ('src.utils.picklefield.PickledObjectField', [], {'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
},
'scripts.scriptdb': {
'Meta': {'object_name': 'ScriptDB'},
'db_date_created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
'db_desc': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'db_interval': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
'db_is_active': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'db_key': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'db_lock_storage': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'db_obj': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['objects.ObjectDB']", 'null': 'True', 'blank': 'True'}),
'db_permissions': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}),
'db_persistent': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'db_repeats': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
'db_start_delay': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'db_typeclass_path': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
}
}
complete_apps = ['scripts']

23
src/typeclasses/models.py
View file

@ -27,12 +27,12 @@ these to create custom managers.
"""
import sys
try:
import cPickle as pickle
except ImportError:
import pickle
#try:
# import cPickle as pickle
#except ImportError:
# import pickle
import traceback
from collections import defaultdict
#from collections import defaultdict
from django.db import models, IntegrityError
from django.conf import settings
@ -41,7 +41,7 @@ from django.contrib.contenttypes.models import ContentType
from src.utils.idmapper.models import SharedMemoryModel
from src.server.caches import get_field_cache, set_field_cache, del_field_cache
from src.server.caches import get_attr_cache, set_attr_cache, del_attr_cache
from src.server.caches import get_prop_cache, set_prop_cache, del_prop_cache
from src.server.caches import get_prop_cache, set_prop_cache, del_prop_cache, flush_attr_cache
from src.server.caches import call_ndb_hooks
from src.server.models import ServerConfig
from src.typeclasses import managers
@ -59,8 +59,8 @@ _CTYPEGET = ContentType.objects.get
_GA = object.__getattribute__
_SA = object.__setattr__
_DA = object.__delattr__
_PLOADS = pickle.loads
_PDUMPS = pickle.dumps
#_PLOADS = pickle.loads
#_PDUMPS = pickle.dumps
#------------------------------------------------------------
#
@ -415,9 +415,8 @@ class Attribute(SharedMemoryModel):
value = from_pickle(self.db_value2, db_obj=self)
self.cached_value = value
self.no_cache = False
else:
# normally the memory cache holds the latest data so no db access is needed.
return self.cached_value
return self.cached_value
#if self.no_cache:
# # re-create data from database and cache it
# try:
@ -1360,7 +1359,7 @@ class TypedObject(SharedMemoryModel):
try:
attrib_obj.value = new_value
except IntegrityError:
# this can happen if the cache was stale and the databse object is
# this can happen if the cache was stale and the database object is
# missing. If so we need to clean self.hashid from the cache
flush_attr_cache(self)
self.delete()

110
src/utils/dbserialize.py
View file

@ -28,6 +28,7 @@ from django.db import transaction
from django.core.exceptions import ObjectDoesNotExist
from django.contrib.contenttypes.models import ContentType
from src.utils.utils import to_str
from src.utils import logger
HIGHEST_PROTOCOL = 2
@ -35,13 +36,21 @@ HIGHEST_PROTOCOL = 2
_GA = object.__getattribute__
_SA = object.__setattr__
_FROM_MODEL_MAP = defaultdict(str)
_FROM_MODEL_MAP.update(dict((c.model, c.natural_key()) for c in ContentType.objects.all()))
_TO_MODEL_MAP = defaultdict(str)
_TO_MODEL_MAP.update(dict((c.natural_key(), c.model_class()) for c in ContentType.objects.all()))
_FROM_MODEL_MAP = None
_TO_MODEL_MAP = None
_TO_TYPECLASS = lambda o: hasattr(o, 'typeclass') and o.typeclass or o
_IS_PACKED_DBOBJ = lambda o: type(o) == tuple and len(o) == 4 and o[0] == '__packed_dbobj__'
_TO_DATESTRING = lambda o: _GA(o, "db_date_created").strftime("%Y:%m:%d-%H:%M:%S:%f")
def _init_globals():
"Lazy importing to avoid circular collisions"
global _FROM_MODEL_MAP, _TO_MODEL_MAP
if not _FROM_MODEL_MAP:
_FROM_MODEL_MAP = defaultdict(str)
_FROM_MODEL_MAP.update(dict((c.model, c.natural_key()) for c in ContentType.objects.all()))
if not _TO_MODEL_MAP:
_TO_MODEL_MAP = defaultdict(str)
_TO_MODEL_MAP.update(dict((c.natural_key(), c.model_class()) for c in ContentType.objects.all()))
#
# SaverList, SaverDict, SaverSet - Attribute-specific helper classes and functions
@ -64,33 +73,39 @@ class SaverMutable(object):
"""
def __init__(self, *args, **kwargs):
"store all properties for tracking the tree"
self._parent = kwargs.pop("parent", None)
self._db_obj = kwargs.pop("db_obj", None)
self._parent = None
self._data = None
def _save_tree(self):
"recursively traverse back up the tree, save when we reach the root"
if self._parent:
self._parent._save_tree()
else:
try:
self._db_obj.value = self
except AttributeError:
raise AttributeError("SaverMutable %s lacks dobj at its root." % self)
def _convert_mutables(self, item):
elif self._db_obj:
self._db_obj.value = self
logger.log_err("SaverMutable %s has no root Attribute to save to." % self)
def _convert_mutables(self, data):
"converts mutables to Saver* variants and assigns .parent property"
dtype = type(item)
if dtype in (basestring, int, long, float, bool, tuple):
def process_tree(item, parent):
"recursively populate the tree, storing parents"
dtype = type(item)
if dtype in (basestring, int, long, float, bool, tuple):
return item
elif dtype == list:
dat = SaverList(parent=parent)
dat._data.extend(process_tree(val, dat) for val in item)
return dat
elif dtype == dict:
dat = SaverDict(parent=parent)
dat._data.update((key, process_tree(val, dat)) for key, val in item.items())
return dat
elif dtype == set:
dat = SaverSet(parent=parent)
dat._data.update(process_tree(val, dat) for val in item)
return dat
return item
elif dtype == list:
item = SaverList(item)
item._parent = self
elif dtype == dict:
item = SaverDict(item)
item._parent = self
elif dtype == set:
item = SaverSet(item)
item._parent = self
return item
return process_tree(data, self)
def __repr__(self):
return self._data.__repr__()
def __len__(self):
@ -106,6 +121,7 @@ class SaverMutable(object):
def __delitem__(self, key):
self._data.__delitem__(key)
class SaverList(SaverMutable, MutableSequence):
"""
A list that saves itself to an Attribute when updated.
@ -149,13 +165,14 @@ class SaverSet(SaverMutable, MutableSet):
def _pack_dbobj(item):
"""
Check and convert django database objects to an internal representation.
This either returns the original input item or a tuple ("__packed_dbobj__", key, obj, id)
This either returns the original input item or a tuple ("__packed_dbobj__", key, creation_time, id)
"""
_init_globals()
obj = hasattr(item, 'dbobj') and item.dbobj or item
natural_key = _FROM_MODEL_MAP[hasattr(obj, "id") and hasattr("db_date_created") and
natural_key = _FROM_MODEL_MAP[hasattr(obj, "id") and hasattr(obj, "db_date_created") and
hasattr(obj, '__class__') and obj.__class__.__name__.lower()]
# build the internal representation as a tuple ("__packed_dbobj__", key, obj, id)
return natural_key and ('__packed_dbobj__', natural_key, _GA(obj, "db_date_created"), _GA(obj, id)) or item
# build the internal representation as a tuple ("__packed_dbobj__", key, creation_time, id)
return natural_key and ('__packed_dbobj__', natural_key, _TO_DATESTRING(obj), _GA(obj, "id")) or item
def _unpack_dbobj(item):
"""
@ -164,12 +181,13 @@ def _unpack_dbobj(item):
This either returns the original input or converts the internal store back
to a database representation (its typeclass is returned if applicable).
"""
_init_globals()
try:
obj = item[3] and _TO_TYPECLASS(_TO_MODEL_MAP[item[1]].objects.get(id=item[3]))
except ObjectDoesNotExist:
return None
# even if we got back a match, check the sanity of the date (some databases may 're-use' the id)
return obj and obj.db_data_created == item[3] and obj or None
return _TO_DATESTRING(obj.dbobj) == item[2] and obj or None
def to_pickle(data):
"""
@ -186,7 +204,7 @@ def to_pickle(data):
elif dtype == tuple:
return tuple(process_item(val) for val in item)
elif dtype in (list, SaverList):
return [key for key in item]
return [process_item(val) for val in item]
elif dtype in (dict, SaverDict):
return dict((key, process_item(val)) for key, val in item.items())
elif dtype in (set, SaverSet):
@ -237,7 +255,7 @@ def from_pickle(data, db_obj=None):
return [process_item(val) for val in item]
return item
def process_item_to_savers(item):
def process_tree(item, parent):
"Recursive processor, convertion and identification of data"
dtype = type(item)
if dtype in (basestring, int, long, float, bool):
@ -246,19 +264,27 @@ def from_pickle(data, db_obj=None):
# this must be checked before tuple
return _unpack_dbobj(item)
elif dtype == tuple:
return tuple(process_item_to_savers(val) for val in item)
return tuple(process_tree(val) for val in item)
elif dtype == list:
return SaverList(process_item_to_savers(val) for val in item)
dat = SaverList(parent=parent)
dat._data.extend(process_tree(val, dat) for val in item)
return dat
elif dtype == dict:
return SaverDict((key, process_item_to_savers(val)) for key, val in item.items())
dat = SaverDict(parent=parent)
dat._data.update(dict((key, process_tree(val, dat)) for key, val in item.items()))
return dat
elif dtype == set:
return SaverSet(process_item_to_savers(val) for val in item)
dat = SaverSet(parent=parent)
dat._data.update(set(process_tree(val, dat) for val in item))
return dat
elif hasattr(item, '__iter__'):
try:
# we try to conserve the iterable class if it accepts an iterator
return item.__class__(process_item_to_savers(val) for val in item)
return item.__class__(process_tree(val, parent) for val in item)
except (AttributeError, TypeError):
return SaverList(process_item_to_savers(val) for val in item)
dat = SaverList(parent=parent)
dat._data.extend(process_tree(val, dat) for val in item)
return dat
return item
if db_obj:
@ -266,11 +292,17 @@ def from_pickle(data, db_obj=None):
# is only relevant if the "root" is an iterable of the right type.
dtype = type(data)
if dtype == list:
return process_item_to_savers(SaverList(data, db_obj=db_obj))
dat = SaverList(db_obj=db_obj)
dat._data.extend(process_tree(val, parent=dat) for val in data)
return dat
elif dtype == dict:
return process_item_to_savers(SaverDict(data, db_obj=db_obj))
dat = SaverDict(db_obj=db_obj)
dat._data.update((key, process_tree(val, parent=dat)) for key, val in data.items())
return dat
elif dtype == set:
return process_item_to_savers(SaverSet(data, db_obj=db_obj))
dat = SaverSet(db_obj=db_obj)
dat._data.update(process_tree(val, parent=dat) for val in data)
return dat
return process_item(data)
def do_pickle(data):