diff --git a/CHANGELOG.md b/CHANGELOG.md index 471aec0b1a..9d131062da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,8 @@ without arguments starts a full interactive Python console. candidates, Builders+ will use list, local search and only global search if no match found. - Make `cmd.at_post_cmd()` always run after `cmd.func()`, even when the latter uses delays with yield. +- Add new `return_iterators` kwarg to `search_prototypes` function in order to prepare for + more paginated handling of prototype returns. ## Evennia 0.9 (2018-2019) diff --git a/evennia/prototypes/prototypes.py b/evennia/prototypes/prototypes.py index 98cfa9505c..9aea89711a 100644 --- a/evennia/prototypes/prototypes.py +++ b/evennia/prototypes/prototypes.py @@ -9,8 +9,11 @@ import hashlib import time from ast import literal_eval from django.conf import settings +from django.db.models import Q, Subquery +from django.core.paginator import Paginator from evennia.scripts.scripts import DefaultScript from evennia.objects.models import ObjectDB +from evennia.typeclasses.attributes import Attribute from evennia.utils.create import create_script from evennia.utils.utils import ( all_from_module, @@ -320,7 +323,7 @@ def delete_prototype(prototype_key, caller=None): return True -def search_prototype(key=None, tags=None, require_single=False): +def search_prototype(key=None, tags=None, require_single=False, return_iterators=False): """ Find prototypes based on key and/or tags, or all prototypes. @@ -331,11 +334,17 @@ def search_prototype(key=None, tags=None, require_single=False): tag category. require_single (bool): If set, raise KeyError if the result was not found or if there are multiple matches. + return_iterators (bool): Optimized return for large numbers of db-prototypes. + If set, separate returns of module based prototypes and paginate + the db-prototype return. Return: - matches (list): All found prototype dicts. Empty list if + matches (list): Default return, all found prototype dicts. Empty list if no match was found. Note that if neither `key` nor `tags` were given, *all* available prototypes will be returned. + list, queryset: If `return_iterators` are found, this is a list of + module-based prototypes followed by a *paginated* queryset of + db-prototypes. Raises: KeyError: If `require_single` is True and there are 0 or >1 matches. @@ -387,27 +396,41 @@ def search_prototype(key=None, tags=None, require_single=False): if key: # exact or partial match on key db_matches = ( - db_matches.filter(db_key=key) or db_matches.filter(db_key__icontains=key) - ).order_by("id") - # return prototype - db_prototypes = [dbprot.prototype for dbprot in db_matches] - - matches = db_prototypes + module_prototypes - nmatches = len(matches) - if nmatches > 1 and key: - key = key.lower() - # avoid duplicates if an exact match exist between the two types - filter_matches = [ - mta for mta in matches if mta.get("prototype_key") and mta["prototype_key"] == key - ] - if filter_matches and len(filter_matches) < nmatches: - matches = filter_matches - - nmatches = len(matches) - if nmatches != 1 and require_single: - raise KeyError("Found {} matching prototypes.".format(nmatches)) - - return matches + db_matches + .filter( + Q(db_key__iexact=key) | Q(db_key__icontains=key)) + .order_by("id") + ) + # convert to prototype + db_ids = db_matches.values_list("id", flat=True) + db_matches = ( + Attribute.objects + .filter(scriptdb__pk__in=db_ids, db_key="prototype") + .values_list("db_value", flat=True) + ) + if key: + matches = list(db_matches) + module_prototypes + nmatches = len(matches) + if nmatches > 1: + key = key.lower() + # avoid duplicates if an exact match exist between the two types + filter_matches = [ + mta for mta in matches if mta.get("prototype_key") and mta["prototype_key"] == key + ] + if filter_matches and len(filter_matches) < nmatches: + matches = filter_matches + nmatches = len(matches) + if nmatches != 1 and require_single: + raise KeyError("Found {} matching prototypes.".format(nmatches)) + return matches + elif return_iterators: + # trying to get the entire set of prototypes - we must paginate + # we must paginate the result of trying to fetch the entire set + db_pages = Paginator(db_matches, 500) + return module_prototypes, db_pages + else: + # full fetch, no pagination + return list(db_matches) + module_prototypes def search_objects_with_prototype(prototype_key): diff --git a/evennia/prototypes/tests.py b/evennia/prototypes/tests.py index edbb5acc51..574013ee4d 100644 --- a/evennia/prototypes/tests.py +++ b/evennia/prototypes/tests.py @@ -3,8 +3,10 @@ Unit tests for the prototypes and spawner """ -from random import randint +from random import randint, sample import mock +import uuid +from time import time from anything import Something from django.test.utils import override_settings from evennia.utils.test_resources import EvenniaTest @@ -1073,3 +1075,29 @@ class TestOLCMenu(TestEvMenu): ["node_index", "node_index", "node_index"], ], ] + +class PrototypeCrashTest(EvenniaTest): + + # increase this to 1000 for optimization testing + num_prototypes = 10 + + def create(self, num=None): + if not num: + num = self.num_prototypes + # print(f"Creating {num} additional prototypes...") + for x in range(num): + prot = { + 'prototype_key': str(uuid.uuid4()), + 'some_attributes': [str(uuid.uuid4()) for x in range(10)], + 'prototype_tags': list(sample(['demo', 'test', 'stuff'], 2)), + } + protlib.save_prototype(prot) + + def test_prototype_dos(self, *args, **kwargs): + num_prototypes = self.num_prototypes + for x in range(2): + self.create(num_prototypes) + # print("Attempting to list prototypes...") + start_time = time() + self.char1.execute_cmd('spawn/list') + # print(f"Prototypes listed in {time()-start_time} seconds.")