From 3579fd0e6135dd83823284ee41920be299661c01 Mon Sep 17 00:00:00 2001
From: Andrew Bastien <volundmush@gmail.com>
Date: Fri, 10 Apr 2020 15:28:13 -0700
Subject: [PATCH 1/6] Add a contents indexer to optimize inventory lookups.

---
 evennia/objects/models.py  | 30 +++++++++++++++++++++---------
 evennia/objects/objects.py | 38 +++++++++++++++++++++++---------------
 2 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/evennia/objects/models.py b/evennia/objects/models.py
index f9e7799d48..b0fdcc801a 100644
--- a/evennia/objects/models.py
+++ b/evennia/objects/models.py
@@ -13,6 +13,7 @@ Attributes are separate objects that store values persistently onto
 the database object. Like everything else, they can be accessed
 transparently through the decorating TypeClass.
 """
+from collections import defaultdict
 from django.conf import settings
 from django.db import models
 from django.core.exceptions import ObjectDoesNotExist
@@ -42,8 +43,9 @@ class ContentsHandler(object):
 
         """
         self.obj = obj
-        self._pkcache = {}
+        self._pkcache = set()
         self._idcache = obj.__class__.__instance_cache__
+        self._typecache = defaultdict(set)
         self.init()
 
     def init(self):
@@ -51,25 +53,30 @@ class ContentsHandler(object):
         Re-initialize the content cache
 
         """
-        self._pkcache.update(
-            dict((obj.pk, None) for obj in ObjectDB.objects.filter(db_location=self.obj) if obj.pk)
-        )
+        objects = [obj for obj in ObjectDB.objects.filter(db_location=self.obj) if obj.pk]
+        self._pkcache = {obj.pk for obj in objects}
+        for obj in objects:
+            for ctype in obj._content_types:
+                self._typecache[ctype].add(obj.pk)
 
-    def get(self, exclude=None):
+    def get(self, exclude=None, category=None):
         """
         Return the contents of the cache.
 
         Args:
             exclude (Object or list of Object): object(s) to ignore
+            category (str or None): Filter list by a content-type. If None, don't filter.
 
         Returns:
             objects (list): the Objects inside this location
 
         """
-        if exclude:
-            pks = [pk for pk in self._pkcache if pk not in [excl.pk for excl in make_iter(exclude)]]
+        if category is not None:
+            pks = self._typecache[category]
         else:
             pks = self._pkcache
+        if exclude:
+            pks = pks - {excl.pk for excl in make_iter(exclude)}
         try:
             return [self._idcache[pk] for pk in pks]
         except KeyError:
@@ -91,7 +98,9 @@ class ContentsHandler(object):
             obj (Object): object to add
 
         """
-        self._pkcache[obj.pk] = None
+        self._pkcache.add(obj.pk)
+        for ctype in obj._content_types:
+            self._typecache[ctype].add(obj.pk)
 
     def remove(self, obj):
         """
@@ -101,7 +110,9 @@ class ContentsHandler(object):
             obj (Object): object to remove
 
         """
-        self._pkcache.pop(obj.pk, None)
+        self._pkcache.remove(obj.pk)
+        for ctype in obj._content_types:
+            self._typecache[ctype].discard(obj.pk)
 
     def clear(self):
         """
@@ -109,6 +120,7 @@ class ContentsHandler(object):
 
         """
         self._pkcache = {}
+        self._typecache = defaultdict(set)
         self.init()
 
 
diff --git a/evennia/objects/objects.py b/evennia/objects/objects.py
index de9d09abae..a0178514d4 100644
--- a/evennia/objects/objects.py
+++ b/evennia/objects/objects.py
@@ -203,6 +203,8 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
     without `obj.save()` having to be called explicitly.
 
     """
+    # Used for sorting / filtering.
+    _content_types = ("object")
 
     # lockstring of newly created objects, for easy overloading.
     # Will be formatted with the appropriate attributes.
@@ -257,7 +259,7 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
             and not self.db_account.attributes.get("_quell")
         )
 
-    def contents_get(self, exclude=None):
+    def contents_get(self, exclude=None, category=None):
         """
         Returns the contents of this object, i.e. all
         objects that has this object set as its location.
@@ -266,6 +268,7 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
         Args:
             exclude (Object): Object to exclude from returned
                 contents list
+            category (str): A category to filter by. None for no filtering.
 
         Returns:
             contents (list): List of contents of this Object.
@@ -1656,20 +1659,25 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
             **kwargs (dict): Arbitrary, optional arguments for users
                 overriding the call (unused by default).
         """
+        def filter_visible(obj_list):
+            # Helper method to determine if objects are visible to the looker.
+            return [obj for obj in obj_list if obj != looker and obj.access(looker, "view")]
+
         if not looker:
             return ""
+
         # get and identify all objects
-        visible = (con for con in self.contents if con != looker and con.access(looker, "view"))
-        exits, users, things = [], [], defaultdict(list)
-        for con in visible:
-            key = con.get_display_name(looker)
-            if con.destination:
-                exits.append(key)
-            elif con.has_account:
-                users.append("|c%s|n" % key)
-            else:
-                # things can be pluralized
-                things[key].append(con)
+        exits_list = filter_visible(self.contents_get(category='exit'))
+        users_list = filter_visible(self.contents_get(category='character'))
+        things_list = filter_visible(self.contents_get(category="object"))
+
+        things = defaultdict(list)
+
+        for thing in things_list:
+            things[thing.key].append(thing)
+        users = [f"|c{user.key}|n" for user in users_list]
+        exits = [ex.key for ex in exits_list]
+
         # get description, build string
         string = "|c%s|n\n" % self.get_display_name(looker)
         desc = self.db.desc
@@ -2026,7 +2034,7 @@ class DefaultCharacter(DefaultObject):
     a character avatar controlled by an account.
 
     """
-
+    _content_types = ("character")
     # lockstring of newly created rooms, for easy overloading.
     # Will be formatted with the appropriate attributes.
     lockstring = "puppet:id({character_id}) or pid({account_id}) or perm(Developer) or pperm(Developer);delete:id({account_id}) or perm(Admin)"
@@ -2239,7 +2247,7 @@ class DefaultRoom(DefaultObject):
     This is the base room object. It's just like any Object except its
     location is always `None`.
     """
-
+    _content_types = ("room", "object")
     # lockstring of newly created rooms, for easy overloading.
     # Will be formatted with the {id} of the creating object.
     lockstring = (
@@ -2389,7 +2397,7 @@ class DefaultExit(DefaultObject):
     exits simply by giving the exit-object's name on its own.
 
     """
-
+    _content_types = ("exit")
     exit_command = ExitCommand
     priority = 101
 

From bd6958d68f9b647a26fde7f990624c166307b0e9 Mon Sep 17 00:00:00 2001
From: Andrew Bastien <volundmush@gmail.com>
Date: Fri, 10 Apr 2020 15:31:55 -0700
Subject: [PATCH 2/6] Add a contents indexer to optimize inventory lookups.

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 92c3758b52..9100761d4a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 - new `drop:holds()` lock default to limit dropping nonsensical things. Access check
   defaults to True for backwards-compatibility in 0.9, will be False in 1.0
+- Added content_types indexing to DefaultObject's ContentsHandler.
 
 ### Already in master
 - `is_typeclass(obj (Object), exact (bool))` now defaults to exact=False

From 2fcb6e9466cddde2722398be522d6dff47658f79 Mon Sep 17 00:00:00 2001
From: Andrew Bastien <volundmush@gmail.com>
Date: Fri, 10 Apr 2020 15:28:13 -0700
Subject: [PATCH 3/6] Add a contents indexer to optimize inventory lookups.

---
 evennia/objects/models.py  | 30 +++++++++++++++++++++---------
 evennia/objects/objects.py | 38 +++++++++++++++++++++++---------------
 2 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/evennia/objects/models.py b/evennia/objects/models.py
index f9e7799d48..b0fdcc801a 100644
--- a/evennia/objects/models.py
+++ b/evennia/objects/models.py
@@ -13,6 +13,7 @@ Attributes are separate objects that store values persistently onto
 the database object. Like everything else, they can be accessed
 transparently through the decorating TypeClass.
 """
+from collections import defaultdict
 from django.conf import settings
 from django.db import models
 from django.core.exceptions import ObjectDoesNotExist
@@ -42,8 +43,9 @@ class ContentsHandler(object):
 
         """
         self.obj = obj
-        self._pkcache = {}
+        self._pkcache = set()
         self._idcache = obj.__class__.__instance_cache__
+        self._typecache = defaultdict(set)
         self.init()
 
     def init(self):
@@ -51,25 +53,30 @@ class ContentsHandler(object):
         Re-initialize the content cache
 
         """
-        self._pkcache.update(
-            dict((obj.pk, None) for obj in ObjectDB.objects.filter(db_location=self.obj) if obj.pk)
-        )
+        objects = [obj for obj in ObjectDB.objects.filter(db_location=self.obj) if obj.pk]
+        self._pkcache = {obj.pk for obj in objects}
+        for obj in objects:
+            for ctype in obj._content_types:
+                self._typecache[ctype].add(obj.pk)
 
-    def get(self, exclude=None):
+    def get(self, exclude=None, category=None):
         """
         Return the contents of the cache.
 
         Args:
             exclude (Object or list of Object): object(s) to ignore
+            category (str or None): Filter list by a content-type. If None, don't filter.
 
         Returns:
             objects (list): the Objects inside this location
 
         """
-        if exclude:
-            pks = [pk for pk in self._pkcache if pk not in [excl.pk for excl in make_iter(exclude)]]
+        if category is not None:
+            pks = self._typecache[category]
         else:
             pks = self._pkcache
+        if exclude:
+            pks = pks - {excl.pk for excl in make_iter(exclude)}
         try:
             return [self._idcache[pk] for pk in pks]
         except KeyError:
@@ -91,7 +98,9 @@ class ContentsHandler(object):
             obj (Object): object to add
 
         """
-        self._pkcache[obj.pk] = None
+        self._pkcache.add(obj.pk)
+        for ctype in obj._content_types:
+            self._typecache[ctype].add(obj.pk)
 
     def remove(self, obj):
         """
@@ -101,7 +110,9 @@ class ContentsHandler(object):
             obj (Object): object to remove
 
         """
-        self._pkcache.pop(obj.pk, None)
+        self._pkcache.remove(obj.pk)
+        for ctype in obj._content_types:
+            self._typecache[ctype].discard(obj.pk)
 
     def clear(self):
         """
@@ -109,6 +120,7 @@ class ContentsHandler(object):
 
         """
         self._pkcache = {}
+        self._typecache = defaultdict(set)
         self.init()
 
 
diff --git a/evennia/objects/objects.py b/evennia/objects/objects.py
index 23d623bfb9..936c173455 100644
--- a/evennia/objects/objects.py
+++ b/evennia/objects/objects.py
@@ -203,6 +203,8 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
     without `obj.save()` having to be called explicitly.
 
     """
+    # Used for sorting / filtering.
+    _content_types = ("object")
 
     # lockstring of newly created objects, for easy overloading.
     # Will be formatted with the appropriate attributes.
@@ -257,7 +259,7 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
             and not self.db_account.attributes.get("_quell")
         )
 
-    def contents_get(self, exclude=None):
+    def contents_get(self, exclude=None, category=None):
         """
         Returns the contents of this object, i.e. all
         objects that has this object set as its location.
@@ -266,6 +268,7 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
         Args:
             exclude (Object): Object to exclude from returned
                 contents list
+            category (str): A category to filter by. None for no filtering.
 
         Returns:
             contents (list): List of contents of this Object.
@@ -1656,20 +1659,25 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
             **kwargs (dict): Arbitrary, optional arguments for users
                 overriding the call (unused by default).
         """
+        def filter_visible(obj_list):
+            # Helper method to determine if objects are visible to the looker.
+            return [obj for obj in obj_list if obj != looker and obj.access(looker, "view")]
+
         if not looker:
             return ""
+
         # get and identify all objects
-        visible = (con for con in self.contents if con != looker and con.access(looker, "view"))
-        exits, users, things = [], [], defaultdict(list)
-        for con in visible:
-            key = con.get_display_name(looker)
-            if con.destination:
-                exits.append(key)
-            elif con.has_account:
-                users.append("|c%s|n" % key)
-            else:
-                # things can be pluralized
-                things[key].append(con)
+        exits_list = filter_visible(self.contents_get(category='exit'))
+        users_list = filter_visible(self.contents_get(category='character'))
+        things_list = filter_visible(self.contents_get(category="object"))
+
+        things = defaultdict(list)
+
+        for thing in things_list:
+            things[thing.key].append(thing)
+        users = [f"|c{user.key}|n" for user in users_list]
+        exits = [ex.key for ex in exits_list]
+
         # get description, build string
         string = "|c%s|n\n" % self.get_display_name(looker)
         desc = self.db.desc
@@ -2026,7 +2034,7 @@ class DefaultCharacter(DefaultObject):
     a character avatar controlled by an account.
 
     """
-
+    _content_types = ("character")
     # lockstring of newly created rooms, for easy overloading.
     # Will be formatted with the appropriate attributes.
     lockstring = "puppet:id({character_id}) or pid({account_id}) or perm(Developer) or pperm(Developer);delete:id({account_id}) or perm(Admin)"
@@ -2277,7 +2285,7 @@ class DefaultRoom(DefaultObject):
     This is the base room object. It's just like any Object except its
     location is always `None`.
     """
-
+    _content_types = ("room", "object")
     # lockstring of newly created rooms, for easy overloading.
     # Will be formatted with the {id} of the creating object.
     lockstring = (
@@ -2427,7 +2435,7 @@ class DefaultExit(DefaultObject):
     exits simply by giving the exit-object's name on its own.
 
     """
-
+    _content_types = ("exit")
     exit_command = ExitCommand
     priority = 101
 

From c031072287e21bd503d35f03474590ea1747e372 Mon Sep 17 00:00:00 2001
From: Andrew Bastien <volundmush@gmail.com>
Date: Fri, 10 Apr 2020 15:31:55 -0700
Subject: [PATCH 4/6] Add a contents indexer to optimize inventory lookups.

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a5272d7c8c..475e9979c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,7 +7,7 @@
 - REST API allows you external access to db objects through HTTP requests (Tehom)
 - `Object.normalize_name` and `.validate_name` added to (by default) enforce latinify
   on character name and avoid potential exploits using clever Unicode chars (trhr)
-
+- Added content_types indexing to DefaultObject's ContentsHandler.
 
 ### Already in master
 - `is_typeclass(obj (Object), exact (bool))` now defaults to exact=False

From 467fd93f122bc6ef9776f519dabb9455e2104bcb Mon Sep 17 00:00:00 2001
From: Andrew Bastien <volundmush@gmail.com>
Date: Sat, 11 Apr 2020 13:34:28 -0700
Subject: [PATCH 5/6] Cleaned up the contents indexer.

---
 evennia/objects/models.py  |  2 +-
 evennia/objects/objects.py | 29 ++++++++++++++++++-----------
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/evennia/objects/models.py b/evennia/objects/models.py
index b0fdcc801a..de51c2c5fe 100644
--- a/evennia/objects/models.py
+++ b/evennia/objects/models.py
@@ -53,7 +53,7 @@ class ContentsHandler(object):
         Re-initialize the content cache
 
         """
-        objects = [obj for obj in ObjectDB.objects.filter(db_location=self.obj) if obj.pk]
+        objects = ObjectDB.objects.filter(db_location=self.obj)
         self._pkcache = {obj.pk for obj in objects}
         for obj in objects:
             for ctype in obj._content_types:
diff --git a/evennia/objects/objects.py b/evennia/objects/objects.py
index 936c173455..85711b4917 100644
--- a/evennia/objects/objects.py
+++ b/evennia/objects/objects.py
@@ -203,8 +203,8 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
     without `obj.save()` having to be called explicitly.
 
     """
-    # Used for sorting / filtering.
-    _content_types = ("object")
+    # Used for sorting / filtering in inventories / room contents.
+    _content_types = ("object",)
 
     # lockstring of newly created objects, for easy overloading.
     # Will be formatted with the appropriate attributes.
@@ -259,7 +259,7 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
             and not self.db_account.attributes.get("_quell")
         )
 
-    def contents_get(self, exclude=None, category=None):
+    def contents_get(self, exclude=None, content_type=None):
         """
         Returns the contents of this object, i.e. all
         objects that has this object set as its location.
@@ -268,13 +268,15 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
         Args:
             exclude (Object): Object to exclude from returned
                 contents list
-            category (str): A category to filter by. None for no filtering.
+            content_type (str): A content_type to filter by. None for no
+                filtering.
 
         Returns:
             contents (list): List of contents of this Object.
 
         Notes:
-            Also available as the `contents` property.
+            Also available as the `contents` property, minus exclusion
+            and filtering.
 
         """
         con = self.contents_cache.get(exclude=exclude)
@@ -1667,9 +1669,9 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
             return ""
 
         # get and identify all objects
-        exits_list = filter_visible(self.contents_get(category='exit'))
-        users_list = filter_visible(self.contents_get(category='character'))
-        things_list = filter_visible(self.contents_get(category="object"))
+        exits_list = filter_visible(self.contents_get(content_type='exit'))
+        users_list = filter_visible(self.contents_get(content_type='character'))
+        things_list = filter_visible(self.contents_get(content_type="object"))
 
         things = defaultdict(list)
 
@@ -2034,7 +2036,9 @@ class DefaultCharacter(DefaultObject):
     a character avatar controlled by an account.
 
     """
-    _content_types = ("character")
+    # Tuple of types used for indexing inventory contents. Characters generally wouldn't be in
+    # anyone's inventory, but this also governs displays in room contents.
+    _content_types = ("character",)
     # lockstring of newly created rooms, for easy overloading.
     # Will be formatted with the appropriate attributes.
     lockstring = "puppet:id({character_id}) or pid({account_id}) or perm(Developer) or pperm(Developer);delete:id({account_id}) or perm(Admin)"
@@ -2285,7 +2289,10 @@ class DefaultRoom(DefaultObject):
     This is the base room object. It's just like any Object except its
     location is always `None`.
     """
-    _content_types = ("room", "object")
+    # A tuple of strings used for indexing this object inside an inventory.
+    # Generally, a room isn't expected to HAVE a location, but maybe in some games?
+    _content_types = ("room",)
+
     # lockstring of newly created rooms, for easy overloading.
     # Will be formatted with the {id} of the creating object.
     lockstring = (
@@ -2435,7 +2442,7 @@ class DefaultExit(DefaultObject):
     exits simply by giving the exit-object's name on its own.
 
     """
-    _content_types = ("exit")
+    _content_types = ("exit",)
     exit_command = ExitCommand
     priority = 101
 

From ff04849031e93fed7090e374c8ddbe6e82265e1b Mon Sep 17 00:00:00 2001
From: Andrew Bastien <volundmush@gmail.com>
Date: Sun, 12 Apr 2020 15:43:29 -0700
Subject: [PATCH 6/6] Fixed some really stupid bugs.

---
 evennia/objects/models.py  | 24 +++++++++++++++++-------
 evennia/objects/objects.py |  4 +---
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/evennia/objects/models.py b/evennia/objects/models.py
index de51c2c5fe..8b42832687 100644
--- a/evennia/objects/models.py
+++ b/evennia/objects/models.py
@@ -48,31 +48,40 @@ class ContentsHandler(object):
         self._typecache = defaultdict(set)
         self.init()
 
+    def load(self):
+        """
+        Retrieves all objects from database. Used for initializing.
+
+        Returns:
+            Objects (list of ObjectDB)
+        """
+        return list(self.obj.locations_set.all())
+
     def init(self):
         """
         Re-initialize the content cache
 
         """
-        objects = ObjectDB.objects.filter(db_location=self.obj)
+        objects = self.load()
         self._pkcache = {obj.pk for obj in objects}
         for obj in objects:
             for ctype in obj._content_types:
                 self._typecache[ctype].add(obj.pk)
 
-    def get(self, exclude=None, category=None):
+    def get(self, exclude=None, content_type=None):
         """
         Return the contents of the cache.
 
         Args:
             exclude (Object or list of Object): object(s) to ignore
-            category (str or None): Filter list by a content-type. If None, don't filter.
+            content_type (str or None): Filter list by a content-type. If None, don't filter.
 
         Returns:
             objects (list): the Objects inside this location
 
         """
-        if category is not None:
-            pks = self._typecache[category]
+        if content_type is not None:
+            pks = self._typecache[content_type]
         else:
             pks = self._pkcache
         if exclude:
@@ -88,7 +97,7 @@ class ContentsHandler(object):
             except KeyError:
                 # this means an actual failure of caching. Return real database match.
                 logger.log_err("contents cache failed for %s." % self.obj.key)
-                return list(ObjectDB.objects.filter(db_location=self.obj))
+                return self.load()
 
     def add(self, obj):
         """
@@ -112,7 +121,8 @@ class ContentsHandler(object):
         """
         self._pkcache.remove(obj.pk)
         for ctype in obj._content_types:
-            self._typecache[ctype].discard(obj.pk)
+            if obj.pk in self._typecache[ctype]:
+                self._typecache[ctype].remove(obj.pk)
 
     def clear(self):
         """
diff --git a/evennia/objects/objects.py b/evennia/objects/objects.py
index 85711b4917..a5fb73f325 100644
--- a/evennia/objects/objects.py
+++ b/evennia/objects/objects.py
@@ -279,9 +279,7 @@ class DefaultObject(ObjectDB, metaclass=TypeclassBase):
             and filtering.
 
         """
-        con = self.contents_cache.get(exclude=exclude)
-        # print "contents_get:", self, con, id(self), calledby()  # DEBUG
-        return con
+        return self.contents_cache.get(exclude=exclude, content_type=content_type)
 
     def contents_set(self, *args):
         "You cannot replace this property"