diff --git a/CHANGELOG.md b/CHANGELOG.md index de062a528a..a6665a90f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,9 +70,11 @@ without arguments starts a full interactive Python console. candidates, Builders+ will use list, local search and only global search if no match found. - Make `cmd.at_post_cmd()` always run after `cmd.func()`, even when the latter uses delays with yield. -- Add new `return_iterators` kwarg to `search_prototypes` function in order to prepare for - more paginated handling of prototype returns. - +- `EvMore` support for db queries and django paginators as well as easier to override for custom + pagination (e.g. to create EvTables for every page instead of splittine one table) +- Using `EvMore pagination`, dramatically improves performance of `spawn/list` and `scripts` listings + (100x speed increase for displaying 1000+ prototypes/scripts). + ## Evennia 0.9 (2018-2019) diff --git a/evennia/commands/default/building.py b/evennia/commands/default/building.py index bdfae2499b..8006130336 100644 --- a/evennia/commands/default/building.py +++ b/evennia/commands/default/building.py @@ -3076,9 +3076,9 @@ class CmdScript(COMMAND_DEFAULT_CLASS): result.append("No scripts defined on %s." % obj.get_display_name(caller)) elif not self.switches: # view all scripts - from evennia.commands.default.system import format_script_list - - result.append(format_script_list(scripts)) + from evennia.commands.default.system import ScriptEvMore + ScriptEvMore(self.caller, scripts.order_by("id"), session=self.session) + return elif "start" in self.switches: num = sum([obj.scripts.start(script.key) for script in scripts]) result.append("%s scripts started on %s." % (num, obj.get_display_name(caller))) @@ -3285,6 +3285,7 @@ class CmdSpawn(COMMAND_DEFAULT_CLASS): spawn/search [prototype_keykey][;tag[,tag]] spawn/list [tag, tag, ...] + spawn/list modules - list only module-based prototypes spawn/show [] spawn/update diff --git a/evennia/commands/default/system.py b/evennia/commands/default/system.py index c121c27c4e..d323cb24e1 100644 --- a/evennia/commands/default/system.py +++ b/evennia/commands/default/system.py @@ -16,6 +16,7 @@ import twisted import time from django.conf import settings +from django.core.paginator import Paginator from evennia.server.sessionhandler import SESSIONS from evennia.scripts.models import ScriptDB from evennia.objects.models import ObjectDB @@ -406,59 +407,71 @@ class CmdPy(COMMAND_DEFAULT_CLASS): ) -# helper function. Kept outside so it can be imported and run -# by other commands. +class ScriptEvMore(EvMore): + """ + Listing 1000+ Scripts can be very slow and memory-consuming. So + we use this custom EvMore child to build en EvTable only for + each page of the list. + """ -def format_script_list(scripts): - """Takes a list of scripts and formats the output.""" - if not scripts: - return "" + def init_pages(self, scripts): + """Prepare the script list pagination""" + script_pages = Paginator(scripts, max(1, int(self.height / 2))) + super().init_pages(script_pages) - table = EvTable( - "|wdbref|n", - "|wobj|n", - "|wkey|n", - "|wintval|n", - "|wnext|n", - "|wrept|n", - "|wdb", - "|wtypeclass|n", - "|wdesc|n", - align="r", - border="tablecols", - ) + def page_formatter(self, scripts): + """Takes a page of scripts and formats the output + into an EvTable.""" - for script in scripts: + if not scripts: + return "" - nextrep = script.time_until_next_repeat() - if nextrep is None: - nextrep = "PAUSED" if script.db._paused_time else "--" - else: - nextrep = "%ss" % nextrep - - maxrepeat = script.repeats - remaining = script.remaining_repeats() or 0 - if maxrepeat: - rept = "%i/%i" % (maxrepeat - remaining, maxrepeat) - else: - rept = "-/-" - - table.add_row( - script.id, - f"{script.obj.key}({script.obj.dbref})" - if (hasattr(script, "obj") and script.obj) - else "", - script.key, - script.interval if script.interval > 0 else "--", - nextrep, - rept, - "*" if script.persistent else "-", - script.typeclass_path.rsplit(".", 1)[-1], - crop(script.desc, width=20), + table = EvTable( + "|wdbref|n", + "|wobj|n", + "|wkey|n", + "|wintval|n", + "|wnext|n", + "|wrept|n", + "|wdb", + "|wtypeclass|n", + "|wdesc|n", + align="r", + border="tablecols", + width=self.width ) - return "%s" % table + for script in scripts: + + nextrep = script.time_until_next_repeat() + if nextrep is None: + nextrep = "PAUSED" if script.db._paused_time else "--" + else: + nextrep = "%ss" % nextrep + + maxrepeat = script.repeats + remaining = script.remaining_repeats() or 0 + if maxrepeat: + rept = "%i/%i" % (maxrepeat - remaining, maxrepeat) + else: + rept = "-/-" + + table.add_row( + script.id, + f"{script.obj.key}({script.obj.dbref})" + if (hasattr(script, "obj") and script.obj) + else "", + script.key, + script.interval if script.interval > 0 else "--", + nextrep, + rept, + "*" if script.persistent else "-", + script.typeclass_path.rsplit(".", 1)[-1], + crop(script.desc, width=20), + ) + + return str(table) class CmdScripts(COMMAND_DEFAULT_CLASS): @@ -547,7 +560,7 @@ class CmdScripts(COMMAND_DEFAULT_CLASS): caller.msg(string) else: # multiple matches. - EvMore(caller, scripts, page_formatter=format_script_list) + ScriptEvMore(caller, scripts, session=self.session) caller.msg("Multiple script matches. Please refine your search") elif self.switches and self.switches[0] in ("validate", "valid", "val"): # run validation on all found scripts @@ -557,7 +570,7 @@ class CmdScripts(COMMAND_DEFAULT_CLASS): caller.msg(string) else: # No stopping or validation. We just want to view things. - EvMore(caller, scripts, page_formatter=format_script_list) + ScriptEvMore(caller, scripts.order_by('id'), session=self.session) class CmdObjects(COMMAND_DEFAULT_CLASS): diff --git a/evennia/commands/default/tests.py b/evennia/commands/default/tests.py index 3fef90e0d9..739f5f9a34 100644 --- a/evennia/commands/default/tests.py +++ b/evennia/commands/default/tests.py @@ -1239,10 +1239,11 @@ class TestBuilding(CommandTest): ) def test_spawn(self): - def getObject(commandTest, objKeyStr): + + def get_object(commandTest, obj_key): # A helper function to get a spawned object and # check that it exists in the process. - query = search_object(objKeyStr) + query = search_object(obj_key) commandTest.assertIsNotNone(query) commandTest.assertTrue(bool(query)) obj = query[0] @@ -1271,7 +1272,7 @@ class TestBuilding(CommandTest): ) self.call(building.CmdSpawn(), "/search ", "Key ") - self.call(building.CmdSpawn(), "/search test;test2", "") + self.call(building.CmdSpawn(), "/search test;test2", "No prototypes found.") self.call( building.CmdSpawn(), @@ -1281,11 +1282,11 @@ class TestBuilding(CommandTest): ) self.call(building.CmdSpawn(), "/list", "Key ") - self.call(building.CmdSpawn(), "testprot", "Spawned Test Char") - # Tests that the spawned object's location is the same as the caharacter's location, since + + # Tests that the spawned object's location is the same as the character's location, since # we did not specify it. - testchar = getObject(self, "Test Char") + testchar = get_object(self, "Test Char") self.assertEqual(testchar.location, self.char1.location) testchar.delete() @@ -1302,7 +1303,7 @@ class TestBuilding(CommandTest): "'key':'goblin', 'location':'%s'}" % spawnLoc.dbref, "Spawned goblin", ) - goblin = getObject(self, "goblin") + goblin = get_object(self, "goblin") # Tests that the spawned object's type is a DefaultCharacter. self.assertIsInstance(goblin, DefaultCharacter) self.assertEqual(goblin.location, spawnLoc) @@ -1321,7 +1322,7 @@ class TestBuilding(CommandTest): # Tests "spawn " self.call(building.CmdSpawn(), "testball", "Spawned Ball") - ball = getObject(self, "Ball") + ball = get_object(self, "Ball") self.assertEqual(ball.location, self.char1.location) self.assertIsInstance(ball, DefaultObject) ball.delete() @@ -1331,7 +1332,7 @@ class TestBuilding(CommandTest): self.call( building.CmdSpawn(), "/n 'BALL'", "Spawned Ball" ) # /n switch is abbreviated form of /noloc - ball = getObject(self, "Ball") + ball = get_object(self, "Ball") self.assertIsNone(ball.location) ball.delete() @@ -1350,7 +1351,7 @@ class TestBuilding(CommandTest): % spawnLoc.dbref, "Spawned Ball", ) - ball = getObject(self, "Ball") + ball = get_object(self, "Ball") self.assertEqual(ball.location, spawnLoc) ball.delete() diff --git a/evennia/prototypes/prototypes.py b/evennia/prototypes/prototypes.py index a6b2965712..79d48cac1d 100644 --- a/evennia/prototypes/prototypes.py +++ b/evennia/prototypes/prototypes.py @@ -391,24 +391,37 @@ def search_prototype(key=None, tags=None, require_single=False, return_iterators # exact match on tag(s) tags = make_iter(tags) tag_categories = ["db_prototype" for _ in tags] - db_matches = DbPrototype.objects.get_by_tag(tags, tag_categories) + db_matches = DbPrototype.objects.get_by_tag( + tags, tag_categories) else: - db_matches = DbPrototype.objects.all().order_by("id") + db_matches = DbPrototype.objects.all() if key: # exact or partial match on key - db_matches = ( + exact_match = ( db_matches .filter( - Q(db_key__iexact=key) | Q(db_key__icontains=key)) - .order_by("id") + Q(db_key__iexact=key)) + .order_by("db_key") ) + if not exact_match: + # try with partial match instead + db_matches = ( + db_matches + .filter( + Q(db_key__icontains=key)) + .order_by("db_key") + ) + else: + db_matches = exact_match + # convert to prototype db_ids = db_matches.values_list("id", flat=True) db_matches = ( Attribute.objects .filter(scriptdb__pk__in=db_ids, db_key="prototype") .values_list("db_value", flat=True) + .order_by("scriptdb__db_key") ) if key and require_single: nmodules = len(module_prototypes) @@ -419,10 +432,9 @@ def search_prototype(key=None, tags=None, require_single=False, return_iterators if return_iterators: # trying to get the entire set of prototypes - we must paginate # the result instead of trying to fetch the entire set at once - db_pages = Paginator(db_matches, 20) - return module_prototypes, db_pages + return db_matches, module_prototypes else: - # full fetch, no pagination + # full fetch, no pagination (compatibility mode) return list(db_matches) + module_prototypes @@ -451,18 +463,45 @@ class PrototypeEvMore(EvMore): """Store some extra properties on the EvMore class""" self.show_non_use = kwargs.pop("show_non_use", False) self.show_non_edit = kwargs.pop("show_non_edit", False) - - # set up table width - width = settings.CLIENT_DEFAULT_WIDTH - if not session: - # fall back to the first session - session = caller.sessions.all()[0] - if session: - width = session.protocol_flags.get("SCREENWIDTH", {0: width})[0] - self.width = width - super().__init__(caller, *args, session=session, **kwargs) + def init_pages(self, inp): + """ + This will be initialized with a tuple (mod_prototype_list, paginated_db_query) + and we must handle these separately since they cannot be paginated in the same + way. We will build the prototypes so that the db-prototypes come first (they + are likely the most volatile), followed by the mod-prototypes. + """ + dbprot_query, modprot_list = inp + # set the number of entries per page to half the reported height of the screen + # to account for long descs etc + dbprot_paged = Paginator(dbprot_query, max(1, int(self.height / 2))) + + # we separate the different types of data, so we track how many pages there are + # of each. + n_mod = len(modprot_list) + self._npages_mod = n_mod // self.height + (0 if n_mod % self.height == 0 else 1) + self._db_count = dbprot_paged.count + self._npages_db = dbprot_paged.num_pages if self._db_count > 0 else 0 + # total number of pages + self._npages = self._npages_mod + self._npages_db + self._data = (dbprot_paged, modprot_list) + self._paginator = self.prototype_paginator + + def prototype_paginator(self, pageno): + """ + The listing is separated in db/mod prototypes, so we need to figure out which + one to pick based on the page number. Also, pageno starts from 0. + """ + dbprot_pages, modprot_list = self._data + + if self._db_count and pageno < self._npages_db: + return dbprot_pages.page(pageno + 1) + else: + # get the correct slice, adjusted for the db-prototypes + pageno = max(0, pageno - self._npages_db) + return modprot_list[pageno * self.height: pageno * self.height + self.height] + def page_formatter(self, page): """Input is a queryset page from django.Paginator""" caller = self._caller @@ -470,7 +509,15 @@ class PrototypeEvMore(EvMore): # get use-permissions of readonly attributes (edit is always False) display_tuples = [] - print("page", page) + table = EvTable( + "|wKey|n", + "|wSpawn/Edit|n", + "|wTags|n", + "|wDesc|n", + border="tablecols", + crop=True, + width=self.width + ) for prototype in page: lock_use = caller.locks.check_lockstring( @@ -490,36 +537,24 @@ class PrototypeEvMore(EvMore): for ptag in prototype.get("prototype_tags", []): if is_iter(ptag): if len(ptag) > 1: - ptags.append("{} (category: {})".format(ptag[0], ptag[1])) + ptags.append("{}".format(ptag[0])) else: ptags.append(ptag[0]) else: ptags.append(str(ptag)) - display_tuples.append( - ( - prototype.get("prototype_key", ""), - "{}/{}".format("Y" if lock_use else "N", "Y" if lock_edit else "N"), - "\n".join(list(set(ptags))), - prototype.get("prototype_desc", ""), - ) + table.add_row( + prototype.get("prototype_key", ""), + "{}/{}".format("Y" if lock_use else "N", "Y" if lock_edit else "N"), + ", ".join(list(set(ptags))), + prototype.get("prototype_desc", ""), ) - if not display_tuples: - return "" - - table = [] - for i in range(len(display_tuples[0])): - table.append([str(display_tuple[i]) for display_tuple in display_tuples]) - table = EvTable("Key", "Spawn/Edit", "Tags", "Desc", table=table, crop=True, width=self.width) - table.reformat_column(0, width=22) - table.reformat_column(1, width=9, align="c") - table.reformat_column(2) - table.reformat_column(3) return str(table) -def list_prototypes(caller, key=None, tags=None, show_non_use=False, show_non_edit=True, session=None): +def list_prototypes(caller, key=None, tags=None, show_non_use=False, + show_non_edit=True, session=None): """ Collate a list of found prototypes based on search criteria and access. @@ -532,33 +567,23 @@ def list_prototypes(caller, key=None, tags=None, show_non_use=False, show_non_ed session (Session, optional): If given, this is used for display formatting. Returns: PrototypeEvMore: An EvMore subclass optimized for prototype listings. - None: If a `key` was given and no matches was found. In this case the caller - has already been notified. + None: If no matches were found. In this case the caller has already been notified. """ # this allows us to pass lists of empty strings tags = [tag for tag in make_iter(tags) if tag] - if key is not None: - matches = search_prototype(key, tags) - if not matches: - caller.msg("No prototypes found.", session=session) - return None - if len(matches) < 2: - matches = [matches] - # get specific prototype (one value or exception) - return PrototypeEvMore(caller, matches, - session=session, - show_non_use=show_non_use, - show_non_edit=show_non_edit) - else: - # list all - # get prototypes for readonly and db-based prototypes - module_prots, db_prots = search_prototype(key, tags, return_iterators=True) - return PrototypeEvMore(caller, db_prots, - session=session, - show_non_use=show_non_use, show_non_edit=show_non_edit) + dbprot_query, modprot_list = search_prototype(key, tags, return_iterators=True) + if not dbprot_query and not modprot_list: + caller.msg("No prototypes found.", session=session) + return None + + # get specific prototype (one value or exception) + return PrototypeEvMore(caller, (dbprot_query, modprot_list), + session=session, + show_non_use=show_non_use, + show_non_edit=show_non_edit) def validate_prototype( prototype, protkey=None, protparents=None, is_prototype_base=True, strict=True, _flags=None diff --git a/evennia/server/portal/amp_server.py b/evennia/server/portal/amp_server.py index de8a08ecf9..d5931f182b 100644 --- a/evennia/server/portal/amp_server.py +++ b/evennia/server/portal/amp_server.py @@ -159,7 +159,7 @@ class AMPServerProtocol(amp.AMPMultiConnectionProtocol): """ # start the Server - print("Portal starting server ... {}".format(server_twistd_cmd)) + print("Portal starting server ... ") process = None with open(settings.SERVER_LOG_FILE, "a") as logfile: # we link stdout to a file in order to catch diff --git a/evennia/utils/evmore.py b/evennia/utils/evmore.py index e623a4fdc5..f992446979 100644 --- a/evennia/utils/evmore.py +++ b/evennia/utils/evmore.py @@ -461,7 +461,7 @@ class EvMore(object): elif not isinstance(inp, str): # anything else not a str self.init_iterable(inp) - self._paginator = self.paginator_index + self._paginator = self.paginator_slice elif "\f" in inp: # string with \f line-break markers in it self.init_f_str(inp) @@ -476,7 +476,7 @@ class EvMore(object): Paginator. The data operated upon is in `self._data`. Args: - pageno (int): The page number to view, from 1...N + pageno (int): The page number to view, from 0...N-1 Returns: str: The page to display (without any decorations, those are added by EvMore).