From f63c155eaf75ff23fc883bdfbd00b8226277cdac Mon Sep 17 00:00:00 2001 From: Griatch Date: Sat, 12 Jun 2021 17:53:24 +0200 Subject: [PATCH] Add caching of pathfinder data --- evennia/contrib/map_and_pathfind/mapsystem.py | 51 +++++++++++++++++-- evennia/contrib/map_and_pathfind/tests.py | 37 +++++++------- evennia/settings_default.py | 5 +- 3 files changed, 71 insertions(+), 22 deletions(-) diff --git a/evennia/contrib/map_and_pathfind/mapsystem.py b/evennia/contrib/map_and_pathfind/mapsystem.py index 48c597d87f..e023e1000a 100644 --- a/evennia/contrib/map_and_pathfind/mapsystem.py +++ b/evennia/contrib/map_and_pathfind/mapsystem.py @@ -81,7 +81,10 @@ See `./example_maps.py` for some empty grid areas to start from. ---- """ +import pickle from collections import defaultdict +from os import mkdir +from os.path import isdir, isfile, join as pathjoin try: from scipy.sparse.csgraph import dijkstra @@ -91,7 +94,11 @@ except ImportError as err: raise ImportError( f"{err}\nThe MapSystem contrib requires " "the SciPy package. Install with `pip install scipy'.") +from django.conf import settings from evennia.utils.utils import variable_from_module, mod_import +from evennia.utils import logger + +_CACHE_DIR = settings.CACHE_DIR _BIG = 999999999999 @@ -1180,6 +1187,10 @@ class Map: Y = y // 2 """ + self.name = name + + mapstring = "" + # store so we can reload self.map_module_or_dict = map_module_or_dict @@ -1197,6 +1208,12 @@ class Map: self.dist_matrix = None self.pathfinding_routes = None + self.pathfinder_baked_filename = None + if name: + if not isdir(_CACHE_DIR): + mkdir(_CACHE_DIR) + self.pathfinder_baked_filename = pathjoin(_CACHE_DIR, f"{name}.P") + # load data and parse it self.reload() @@ -1262,13 +1279,32 @@ class Map: def _calculate_path_matrix(self): """ - Solve the pathfinding problem using Dijkstra's algorithm. + Solve the pathfinding problem using Dijkstra's algorithm. This will try to + load the solution from disk if possible. """ - nnodes = len(self.node_index_map) + if self.pathfinder_baked_filename and isfile(self.pathfinder_baked_filename): + # check if the solution for this grid was already solved previously. + + mapstr, dist_matrix, pathfinding_routes = "", None, None + with open(self.pathfinder_baked_filename, 'rb') as fil: + try: + mapstr, dist_matrix, pathfinding_routes = pickle.load(fil) + except Exception: + logger.log_trace() + if (mapstr == self.mapstring + and dist_matrix is not None + and pathfinding_routes is not None): + # this is important - it means the map hasn't changed so + # we can re-use the stored data! + self.dist_matrix = dist_matrix + self.pathfinding_routes = pathfinding_routes + return - pathfinding_graph = zeros((nnodes, nnodes)) # build a matrix representing the map graph, with 0s as impassable areas + + nnodes = len(self.node_index_map) + pathfinding_graph = zeros((nnodes, nnodes)) for inode, node in self.node_index_map.items(): pathfinding_graph[inode, :] = node.linkweights(nnodes) @@ -1280,6 +1316,12 @@ class Map: pathfinding_matrix, directed=True, return_predecessors=True, limit=self.max_pathfinding_length) + if self.pathfinder_baked_filename: + # try to cache the results + with open(self.pathfinder_baked_filename, 'wb') as fil: + pickle.dump((self.mapstring, self.dist_matrix, self.pathfinding_routes), + fil, protocol=4) + def _parse(self): """ Parses the numerical grid from the string. The result of this is a 2D array @@ -1665,7 +1707,8 @@ class Map: # stylize path to target def _default_callable(node): - return target_path_style.format(display_symbol=node) + return target_path_style.format( + display_symbol=node.get_display_symbol(self.xygrid)) if callable(target_path_style): _target_path_style = target_path_style diff --git a/evennia/contrib/map_and_pathfind/tests.py b/evennia/contrib/map_and_pathfind/tests.py index a42c59e329..569626fe48 100644 --- a/evennia/contrib/map_and_pathfind/tests.py +++ b/evennia/contrib/map_and_pathfind/tests.py @@ -320,7 +320,7 @@ class TestMap1(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP1}) + self.map = mapsystem.Map({"map": MAP1}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -404,7 +404,7 @@ class TestMap2(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP2}) + self.map = mapsystem.Map({"map": MAP2}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -514,7 +514,7 @@ class TestMap3(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP3}) + self.map = mapsystem.Map({"map": MAP3}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -563,7 +563,7 @@ class TestMap4(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP4}) + self.map = mapsystem.Map({"map": MAP4}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -593,7 +593,7 @@ class TestMap5(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP5}) + self.map = mapsystem.Map({"map": MAP5}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -621,7 +621,7 @@ class TestMap6(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP6}) + self.map = mapsystem.Map({"map": MAP6}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -653,7 +653,7 @@ class TestMap7(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP7}) + self.map = mapsystem.Map({"map": MAP7}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -681,7 +681,7 @@ class TestMap8(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP8}) + self.map = mapsystem.Map({"map": MAP8}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -747,7 +747,7 @@ class TestMap9(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP9}) + self.map = mapsystem.Map({"map": MAP9}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -776,7 +776,7 @@ class TestMap10(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP10}) + self.map = mapsystem.Map({"map": MAP10}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -824,7 +824,7 @@ class TestMap11(TestCase): """ def setUp(self): - self.map = mapsystem.Map({"map": MAP11}) + self.map = mapsystem.Map({"map": MAP11}, name="testmap") def test_str_output(self): """Check the display_map""" @@ -914,14 +914,14 @@ class TestMapStressTest(TestCase): grid = self._get_grid(Xmax, Ymax) # print(f"\n\n{grid}\n") t0 = time() - mapsystem.Map({'map': grid}) + mapsystem.Map({'map': grid}, name="testmap") t1 = time() self.assertLess(t1 - t0, max_time, f"Map creation of ({Xmax}x{Ymax}) grid slower " f"than expected {max_time}s.") @parameterized.expand([ - ((10, 10), 10**-4), - ((20, 20), 10**-4), + ((10, 10), 10**-3), + ((20, 20), 10**-3), ]) def test_grid_pathfind(self, gridsize, max_time): """ @@ -930,7 +930,7 @@ class TestMapStressTest(TestCase): """ Xmax, Ymax = gridsize grid = self._get_grid(Xmax, Ymax) - mapobj = mapsystem.Map({'map': grid}) + mapobj = mapsystem.Map({'map': grid}, name="testmap") t0 = time() mapobj._calculate_path_matrix() @@ -962,7 +962,7 @@ class TestMapStressTest(TestCase): """ Xmax, Ymax = gridsize grid = self._get_grid(Xmax, Ymax) - mapobj = mapsystem.Map({'map': grid}) + mapobj = mapsystem.Map({'map': grid}, name="testmap") t0 = time() mapobj._calculate_path_matrix() @@ -978,8 +978,11 @@ class TestMapStressTest(TestCase): t0 = time() for coord, target in start_end_points: - mapobj.get_visual_range(coord, dist=dist, mode='nodes', character='@', target=target) + mapobj.get_visual_range(coord, dist=dist, mode='nodes', + character='@', target=target) t1 = time() self.assertLess((t1 - t0) / 10, max_time, f"Visual Range calculation for ({Xmax}x{Ymax}) grid " f"slower than expected {max_time}s.") + + diff --git a/evennia/settings_default.py b/evennia/settings_default.py index 1450fc6c52..4e07be8789 100644 --- a/evennia/settings_default.py +++ b/evennia/settings_default.py @@ -134,7 +134,6 @@ else: GAME_DIR = gpath break os.chdir(os.pardir) - # Place to put log files, how often to rotate the log and how big each log file # may become before rotating. LOG_DIR = os.path.join(GAME_DIR, "server", "logs") @@ -152,6 +151,10 @@ LOCKWARNING_LOG_FILE = os.path.join(LOG_DIR, "lockwarnings.log") CHANNEL_LOG_NUM_TAIL_LINES = 20 # Max size (in bytes) of channel log files before they rotate CHANNEL_LOG_ROTATE_SIZE = 1000000 +# Unused by default, but used by e.g. the MapSystem contrib. A place for storing +# semi-permanent data and avoid it being rebuilt over and over. It is created +# on-demand only. +CACHE_DIR = os.path.join(GAME_DIR, "server", ".cache") # Local time zone for this installation. All choices can be found here: # http://www.postgresql.org/docs/8.0/interactive/datetime-keywords.html#DATETIME-TIMEZONE-SET-TABLE TIME_ZONE = "UTC"