diff --git a/CHANGELOG.md b/CHANGELOG.md index bc4144d0b2..1fe1e8f273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,6 +207,9 @@ Up requirements to Django 4.0+, Twisted 22+, Python 3.9 or 3.10 powerful searches passed into the regular search functions. - `spawner.spawn` and linked methods now has a kwarg `protfunc_raise_errors` (default True) to disable strict errors on malformed/not-found protfuncs +- Improve search performance when having many DB-based prototypes via caching. +- Remove the `return_parents` kwarg of `evennia.prototypes.spawner.spawn` since it + was inefficient and unused. ## Evennia 0.9.5 diff --git a/evennia/prototypes/prototypes.py b/evennia/prototypes/prototypes.py index d20d53b6fc..b1155d7208 100644 --- a/evennia/prototypes/prototypes.py +++ b/evennia/prototypes/prototypes.py @@ -21,9 +21,15 @@ from evennia.utils.create import create_script from evennia.utils.evmore import EvMore from evennia.utils.evtable import EvTable from evennia.utils.funcparser import FuncParser -from evennia.utils.utils import (all_from_module, class_from_module, - dbid_to_obj, is_iter, justify, make_iter, - variable_from_module) +from evennia.utils.utils import ( + all_from_module, + class_from_module, + dbid_to_obj, + is_iter, + justify, + make_iter, + variable_from_module, +) _MODULE_PROTOTYPE_MODULES = {} _MODULE_PROTOTYPES = {} @@ -57,23 +63,6 @@ _PROTOTYPE_FALLBACK_LOCK = "spawn:all();edit:all()" FUNC_PARSER = FuncParser(settings.PROT_FUNC_MODULES) -class DBPrototypeCache: - def __init__(self): - self._cache = {} - - def get(self, db_prot_id): - return self._cache.get(db_prot_id, None) - - def add(self, db_prot_id, prototype): - self._cache[db_prot_id] = prototype - - def remove(self, db_prot_id): - self._cache.pop(db_prot_id, None) - - -DB_PROTOTYPE_CACHE = DBPrototypeCache() - - class PermissionError(RuntimeError): pass @@ -312,7 +301,7 @@ def load_module_prototypes(*mod_or_prototypes, override=True): prototype_key = mod_or_dict.get("prototype_key") if not prototype_key: raise ValidationError( - f"The prototype {mod_or_prototype} does not contain a 'prototype_key'" + f"The prototype {mod_or_dict} does not contain a 'prototype_key'" ) prots = [(prototype_key, mod_or_dict)] mod = None @@ -341,6 +330,36 @@ def load_module_prototypes(*mod_or_prototypes, override=True): # Db-based prototypes +class DBPrototypeCache: + """ + Cache DB-stored prototypes; it can still be slow to initially load 1000s of + prototypes, due to having to deserialize all prototype-dicts, but after the + first time the cache will be populated and things will be fast. + + """ + + def __init__(self): + self._cache = {} + + def get(self, db_prot_id): + return self._cache.get(db_prot_id, None) + + def add(self, db_prot_id, prototype): + self._cache[db_prot_id] = prototype + + def remove(self, db_prot_id): + self._cache.pop(db_prot_id, None) + + def clear(self): + self._cache = {} + + def replace(self, all_data): + self._cache = all_data + + +DB_PROTOTYPE_CACHE = DBPrototypeCache() + + class DbPrototype(DefaultScript): """ This stores a single prototype, in an Attribute `prototype`. @@ -450,7 +469,7 @@ def save_prototype(prototype): tags=in_prototype["prototype_tags"], attributes=[("prototype", in_prototype)], ) - DB_PROTOTYPE_CACHE.add(stored_prototype.prototype) + DB_PROTOTYPE_CACHE.add(stored_prototype.id, stored_prototype.prototype) return stored_prototype.prototype @@ -495,13 +514,19 @@ def delete_prototype(prototype_key, caller=None): "delete prototype {prototype_key}." ).format(caller=caller, prototype_key=prototype_key) ) - DB_PROTOTYPE_CACHE.remove(stored_prototype.prototype) + DB_PROTOTYPE_CACHE.remove(stored_prototype.id) stored_prototype.delete() return True def search_prototype( - key=None, tags=None, require_single=False, return_iterators=False, no_db=False + key=None, + tags=None, + require_single=False, + return_iterators=False, + no_db=False, + page_size=None, + page_no=None, ): """ Find prototypes based on key and/or tags, or all prototypes. @@ -525,7 +550,7 @@ def search_prototype( no match was found. Note that if neither `key` nor `tags` were given, *all* available prototypes will be returned. list, queryset: If `return_iterators` are found, this is a list of - module-based prototypes followed by a *paginated* queryset of + module-based prototypes followed by a queryset of db-prototypes. Raises: @@ -538,104 +563,117 @@ def search_prototype( be found as a match. """ - # This will load the prototypes the first time they are searched - loaded = getattr(load_module_prototypes, "_LOADED", False) - if not loaded: - load_module_prototypes() - setattr(load_module_prototypes, "_LOADED", True) - # prototype keys are always in lowecase - if key: - key = key.lower() + def _search_module_based_prototypes(key, tags): + """ + Helper function to load module-based prots. - # search module prototypes + """ + # This will load the prototypes the first time they are searched + loaded = getattr(load_module_prototypes, "_LOADED", False) + if not loaded: + load_module_prototypes() + setattr(load_module_prototypes, "_LOADED", True) - mod_matches = {} - if tags: - # use tags to limit selection - tagset = set(tags) - mod_matches = { - prototype_key: prototype - for prototype_key, prototype in _MODULE_PROTOTYPES.items() - if tagset.intersection(prototype.get("prototype_tags", [])) - } - else: - mod_matches = _MODULE_PROTOTYPES + # search module prototypes - allow_fuzzy = True - if key: - if key in mod_matches: - # exact match - module_prototypes = [mod_matches[key].copy()] - allow_fuzzy = False + mod_matches = {} + if tags: + # use tags to limit selection + tagset = set(tags) + mod_matches = { + prototype_key: prototype + for prototype_key, prototype in _MODULE_PROTOTYPES.items() + if tagset.intersection(prototype.get("prototype_tags", [])) + } else: - # fuzzy matching - module_prototypes = [ - prototype - for prototype_key, prototype in mod_matches.items() - if key in prototype_key - ] - else: - # note - we return a copy of the prototype dict, otherwise using this with e.g. - # prototype_from_object will modify the base prototype for every object - module_prototypes = [match.copy() for match in mod_matches.values()] + mod_matches = _MODULE_PROTOTYPES - if no_db: - db_matches = [] - else: + fuzzy_match_db = True + if key: + if key in mod_matches: + # exact match + module_prototypes = [mod_matches[key].copy()] + fuzzy_match_db = False + else: + # fuzzy matching + module_prototypes = [ + prototype + for prototype_key, prototype in mod_matches.items() + if key in prototype_key + ] + else: + # note - we return a copy of the prototype dict, otherwise using this with e.g. + # prototype_from_object will modify the base prototype for every object + module_prototypes = [match.copy() for match in mod_matches.values()] + + return module_prototypes, fuzzy_match_db + + def _search_db_based_prototypes(key, tags, fuzzy_matching): + """ + Helper function for loading db-based prots. + + """ # search db-stored prototypes if tags: # exact match on tag(s) tags = make_iter(tags) tag_categories = ["db_prototype" for _ in tags] - db_matches = DbPrototype.objects.get_by_tag(tags, tag_categories) + query = DbPrototype.objects.get_by_tag(tags, tag_categories) else: - db_matches = DbPrototype.objects.all() + query = DbPrototype.objects.all() if key: # exact or partial match on key - exact_match = db_matches.filter(Q(db_key__iexact=key)).order_by("db_key") - if not exact_match and allow_fuzzy: + exact_match = query.filter(Q(db_key__iexact=key)) + if not exact_match and fuzzy_matching: # try with partial match instead - db_matches = db_matches.filter(Q(db_key__icontains=key)).order_by("db_key") + query = query.filter(Q(db_key__icontains=key)) else: - db_matches = exact_match - db_ids = db_matches.values_list("id", flat=True) - db_matches = Attribute.objects.filter(scriptdb__pk__in=db_ids, db_key="prototype") - .values_list("db_value", flat=True) - .order_by("scriptdb__db_key") + query = exact_match + # convert to prototype, cached or from db - db_protkeys = db_matches.values_list("db_key", flat=True) - # convert to prototype - cache = DB_PROTOTYPE_CACHE.get() - db_matches = [cache.get(protkey) for protkey in db_protkeys if protkey in cache] - else: - # fetch and deserialize all data - db_ids = db_matches.values_list("id", flat=True) - db_matches = ( - Attribute.objects.filter(scriptdb__pk__in=db_ids, db_key="prototype") + db_matches = [] + not_found = [] + for db_id in query.values_list("id", flat=True).order_by("db_key"): + prot = DB_PROTOTYPE_CACHE.get(db_id) + if prot: + db_matches.append(prot) + else: + not_found.append(db_id) + + if not_found: + new_db_matches = ( + Attribute.objects.filter(scriptdb__pk__in=not_found, db_key="prototype") .values_list("db_value", flat=True) .order_by("scriptdb__db_key") ) + for db_id, prot in zip(not_found, new_db_matches): + DB_PROTOTYPE_CACHE.add(db_id, prot) + db_matches.extend(list(new_db_matches)) + + return db_matches + + if key: + key = key.lower() + + module_prototypes, fuzzy_match_db = _search_module_based_prototypes(key, tags) + + db_prototypes = [] if no_db else _search_db_based_prototypes(key, tags, fuzzy_match_db) if key and require_single: - nmodules = len(module_prototypes) - ndbprots = db_matches.count() if db_matches else 0 - if nmodules + ndbprots != 1: - raise KeyError( - _("Found {num} matching prototypes among {module_prototypes}.").format( - num=nmodules + ndbprots, module_prototypes=module_prototypes - ) - ) + num = len(module_prototypes) + len(db_prototypes) + if num != 1: + raise KeyError(_(f"Found {num} matching prototypes.")) if return_iterators: # trying to get the entire set of prototypes - we must paginate # the result instead of trying to fetch the entire set at once - return db_matches, module_prototypes + return db_prototypes, module_prototypes else: # full fetch, no pagination (compatibility mode) - return list(db_matches) + module_prototypes + return list(db_prototypes) + module_prototypes def search_objects_with_prototype(prototype_key): @@ -686,7 +724,7 @@ class PrototypeEvMore(EvMore): # of each. n_mod = len(modprot_list) self._npages_mod = n_mod // self.height + (0 if n_mod % self.height == 0 else 1) - self._db_count = dbprot_paged.count + self._db_count = dbprot_paged.count if dbprot_paged else 0 self._npages_db = dbprot_paged.num_pages if self._db_count > 0 else 0 # total number of pages self._npages = self._npages_mod + self._npages_db @@ -783,7 +821,7 @@ def list_prototypes( dbprot_query, modprot_list = search_prototype(key, tags, return_iterators=True) - if not dbprot_query.count() and not modprot_list: + if not dbprot_query and not modprot_list: caller.msg(_("No prototypes found."), session=session) return None @@ -807,8 +845,9 @@ def validate_prototype( prototype (dict): Prototype to validate. protkey (str, optional): The name of the prototype definition. If not given, the prototype dict needs to have the `prototype_key` field set. - protpartents (dict, optional): The available prototype parent library. If - note given this will be determined from settings/database. + protparents (dict, optional): Additional prototype-parents, supposedly provided specifically + for this prototype. If given, matching parents will first be taken from this + dict rather than from the global set of prototypes found via settings/database. is_prototype_base (bool, optional): We are trying to create a new object *based on this object*. This means we can't allow 'mixin'-style prototypes without typeclass/parent etc. @@ -822,16 +861,11 @@ def validate_prototype( """ assert isinstance(prototype, dict) + protparents = {} if protparents is None else protparents if _flags is None: _flags = {"visited": [], "depth": 0, "typeclass": False, "errors": [], "warnings": []} - if not protparents: - protparents = { - prototype.get("prototype_key", "").lower(): prototype - for prototype in search_prototype() - } - protkey = protkey and protkey.lower() or prototype.get("prototype_key", None) if strict and not bool(protkey): @@ -883,13 +917,20 @@ def validate_prototype( _flags["errors"].append( _("Prototype {protkey} tries to parent itself.").format(protkey=protkey) ) + + # get prototype parent, first try custom set, then search globally protparent = protparents.get(protstring) if not protparent: - _flags["errors"].append( - _( - "Prototype {protkey}'s `prototype_parent` (named '{parent}') was not found." - ).format(protkey=protkey, parent=protstring) - ) + protparent = search_prototype(key=protstring, require_single=True) + if protparent: + protparent = protparent[0] + else: + _flags["errors"].append( + _( + "Prototype {protkey}'s `prototype_parent` (named '{parent}') was not" + " found." + ).format(protkey=protkey, parent=protstring) + ) # check for infinite recursion if id(prototype) in _flags["visited"]: @@ -906,7 +947,11 @@ def validate_prototype( # next step of recursive validation validate_prototype( - protparent, protstring, protparents, is_prototype_base=is_prototype_base, _flags=_flags + protparent, + protkey=protstring, + protparents=protparents, + is_prototype_base=is_prototype_base, + _flags=_flags, ) _flags["visited"].pop() @@ -967,7 +1012,8 @@ def protfunc_parser( available_functions (dict, optional): Mapping of name:protfunction to use for this parsing. If not set, use default sources. stacktrace (bool, optional): If set, print the stack parsing process of the protfunc-parser. - raise_errors (bool, optional): Raise explicit errors from malformed/not found protfunc calls. + raise_errors (bool, optional): Raise explicit errors from malformed/not found protfunc + calls. Keyword Args: session (Session): Passed to protfunc. Session of the entity spawning the prototype. @@ -1117,8 +1163,10 @@ def check_permission(prototype_key, action, default=True): logger.log_err(err.format(protkey=prototype_key, module=mod)) return False - prototype = search_prototype(key=prototype_key) - if not prototype: + prototype = search_prototype(key=prototype_key, require_single=True) + if prototype: + prototype = prototype[0] + else: logger.log_err("Prototype {} not found.".format(prototype_key)) return False diff --git a/evennia/prototypes/spawner.py b/evennia/prototypes/spawner.py index c4ba3cbf75..ffd66c9ae5 100644 --- a/evennia/prototypes/spawner.py +++ b/evennia/prototypes/spawner.py @@ -145,6 +145,7 @@ from evennia.prototypes import prototypes as protlib from evennia.prototypes.prototypes import ( PROTOTYPE_TAG_CATEGORY, init_spawn_value, + search_prototype, value_to_obj, value_to_obj_or_any, ) @@ -190,7 +191,7 @@ class Unset: # Helper -def _get_prototype(inprot, protparents, uninherited=None, _workprot=None): +def _get_prototype(inprot, protparents=None, uninherited=None, _workprot=None): """ Recursively traverse a prototype dictionary, including multiple inheritance. Use validate_prototype before this, we don't check @@ -198,7 +199,9 @@ def _get_prototype(inprot, protparents, uninherited=None, _workprot=None): Args: inprot (dict): Prototype dict (the individual prototype, with no inheritance included). - protparents (dict): Available protparents, keyed by prototype_key. + protparents (dict): Custom protparents, supposedly provided specifically for this `inprot`. + If given, any parents will first be looked up in this dict, and then by searching + the global prototype store given by settings/db. uninherited (dict): Parts of prototype to not inherit. _workprot (dict, optional): Work dict for the recursive algorithm. @@ -220,6 +223,8 @@ def _get_prototype(inprot, protparents, uninherited=None, _workprot=None): old.update(new) return list(old.values()) + protparents = {} if protparents is None else protparents + _workprot = {} if _workprot is None else _workprot if "prototype_parent" in inprot: # move backwards through the inheritance @@ -234,8 +239,12 @@ def _get_prototype(inprot, protparents, uninherited=None, _workprot=None): # protparent already embedded as-is parent_prototype = prototype else: - # protparent given by-name - parent_prototype = protparents.get(prototype.lower(), {}) + # protparent given by-name, first search provided parents, then global store + parent_prototype = protparents.get(prototype.lower()) + if not parent_prototype: + parent_prototype = search_prototype(key=prototype.lower()) or {} + if parent_prototype: + parent_prototype = parent_prototype[0] # Build the prot dictionary in reverse order, overloading new_prot = _get_prototype(parent_prototype, protparents, _workprot=_workprot) @@ -277,14 +286,9 @@ def flatten_prototype(prototype, validate=False, no_db=False): if prototype: prototype = protlib.homogenize_prototype(prototype) - protparents = { - prot["prototype_key"].lower(): prot for prot in protlib.search_prototype(no_db=no_db) - } - protlib.validate_prototype( - prototype, None, protparents, is_prototype_base=validate, strict=validate - ) + protlib.validate_prototype(prototype, is_prototype_base=validate, strict=validate) return _get_prototype( - prototype, protparents, uninherited={"prototype_key": prototype.get("prototype_key")} + prototype, uninherited={"prototype_key": prototype.get("prototype_key")} ) return {} @@ -661,6 +665,8 @@ def batch_update_objects_with_prototype( if isinstance(prototype, str): new_prototype = protlib.search_prototype(prototype) + if new_prototype: + new_prototype = new_prototype[0] else: new_prototype = prototype @@ -892,10 +898,6 @@ def spawn(*prototypes, caller=None, **kwargs): prototype_parents (dict): A dictionary holding a custom prototype-parent dictionary. Will overload same-named prototypes from prototype_modules. - return_parents (bool): Return a dict of the entire prototype-parent tree - available to this prototype (no object creation happens). This is a - merged result between the globally found protparents and whatever - custom `prototype_parents` are given to this function. only_validate (bool): Only run validation of prototype/parents (no object creation) and return the create-kwargs. protfunc_raise_errors (bool): Raise explicit exceptions on a malformed/not-found @@ -903,8 +905,7 @@ def spawn(*prototypes, caller=None, **kwargs): Returns: object (Object, dict or list): Spawned object(s). If `only_validate` is given, return - a list of the creation kwargs to build the object(s) without actually creating it. If - `return_parents` is set, instead return dict of prototype parents. + a list of the creation kwargs to build the object(s) without actually creating it. """ # search string (=prototype_key) from input @@ -913,9 +914,6 @@ def spawn(*prototypes, caller=None, **kwargs): for prot in prototypes ] - # get available protparents - protparents = {prot["prototype_key"].lower(): prot for prot in protlib.search_prototype()} - if not kwargs.get("only_validate"): # homogenization to be more lenient about prototype format when entering the prototype # manually @@ -924,21 +922,23 @@ def spawn(*prototypes, caller=None, **kwargs): # overload module's protparents with specifically given protparents # we allow prototype_key to be the key of the protparent dict, to allow for module-level # prototype imports. We need to insert prototype_key in this case + custom_protparents = {} for key, protparent in kwargs.get("prototype_parents", {}).items(): key = str(key).lower() protparent["prototype_key"] = str(protparent.get("prototype_key", key)).lower() - protparents[key] = protlib.homogenize_prototype(protparent) - - if "return_parents" in kwargs: - # only return the parents - return copy.deepcopy(protparents) + custom_protparents[key] = protlib.homogenize_prototype(protparent) objsparams = [] for prototype in prototypes: - protlib.validate_prototype(prototype, None, protparents, is_prototype_base=True) + # run validation and homogenization of provided prototypes + protlib.validate_prototype( + prototype, None, protparents=custom_protparents, is_prototype_base=True + ) prot = _get_prototype( - prototype, protparents, uninherited={"prototype_key": prototype.get("prototype_key")} + prototype, + protparents=custom_protparents, + uninherited={"prototype_key": prototype.get("prototype_key")}, ) if not prot: continue