From da48fa2e529c801567e1d96b3c8746deb411248a Mon Sep 17 00:00:00 2001 From: Greg Taylor Date: Sun, 15 Sep 2019 18:21:33 -0700 Subject: [PATCH 1/2] Refactor mod_import to use importlib Switch from the deprecated imp to importlib. Also add tests and clean up logic flow. This should be quite a bit faster than the old implementation as well. --- evennia/utils/tests/test_utils.py | 26 ++++++++++++ evennia/utils/utils.py | 68 +++++++++++++++---------------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/evennia/utils/tests/test_utils.py b/evennia/utils/tests/test_utils.py index 36b32318b3..699fdb7756 100644 --- a/evennia/utils/tests/test_utils.py +++ b/evennia/utils/tests/test_utils.py @@ -5,6 +5,8 @@ TODO: Not nearly all utilities are covered yet. """ +import os.path + import mock from django.test import TestCase from datetime import datetime @@ -203,3 +205,27 @@ class TestDateTimeFormat(TestCase): self.assertEqual(utils.datetime_format(dtobj), "19:54") dtobj = datetime(2019, 8, 28, 21, 32) self.assertEqual(utils.datetime_format(dtobj), "21:32:00") + + +class TestImportFunctions(TestCase): + def _t_dir_file(self, filename): + testdir = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(testdir, filename) + + def test_mod_import(self): + loaded_mod = utils.mod_import('evennia.utils.ansi') + self.assertIsNotNone(loaded_mod) + + def test_mod_import_invalid(self): + loaded_mod = utils.mod_import('evennia.utils.invalid_module') + self.assertIsNone(loaded_mod) + + def test_mod_import_from_path(self): + test_path = self._t_dir_file('test_eveditor.py') + loaded_mod = utils.mod_import_from_path(test_path) + self.assertIsNotNone(loaded_mod) + + def test_mod_import_from_path_invalid(self): + test_path = self._t_dir_file('invalid_filename.py') + loaded_mod = utils.mod_import_from_path(test_path) + self.assertIsNone(loaded_mod) diff --git a/evennia/utils/utils.py b/evennia/utils/utils.py index 88b69258d1..ad2a500eab 100644 --- a/evennia/utils/utils.py +++ b/evennia/utils/utils.py @@ -9,7 +9,6 @@ be of use when designing your own game. import os import gc import sys -import imp import types import math import re @@ -17,6 +16,7 @@ import textwrap import random import inspect import traceback +import importlib.machinery from twisted.internet.task import deferLater from twisted.internet.defer import returnValue # noqa - used as import target from os.path import join as osjoin @@ -1166,6 +1166,30 @@ def has_parent(basepath, obj): return False +def mod_import_from_path(path): + """ + Load a Python module at the specified path. + + Args: + path (str): An absolute path to a Python module to load. + + Returns: + (module or None): An imported module if the path was a valid + Python module. Returns `None` if the import failed. + + """ + if not os.path.isabs(path): + path = os.path.abspath(path) + dirpath, filename = path.rsplit(os.path.sep, 1) + modname = filename.rstrip('.py') + + try: + return importlib.machinery.SourceFileLoader(modname, path).load_module() + except OSError: + logger.log_trace(f"Could not find module '{modname}' ({modname}.py) at path '{dirpath}'") + return None + + def mod_import(module): """ A generic Python module loader. @@ -1173,52 +1197,28 @@ def mod_import(module): Args: module (str, module): This can be either a Python path (dot-notation like `evennia.objects.models`), an absolute path - (e.g. `/home/eve/evennia/evennia/objects.models.py`) or an + (e.g. `/home/eve/evennia/evennia/objects/models.py`) or an already imported module object (e.g. `models`) Returns: - module (module or None): An imported module. If the input argument was + (module or None): An imported module. If the input argument was already a module, this is returned as-is, otherwise the path is parsed and imported. Returns `None` and logs error if import failed. """ - if not module: return None if isinstance(module, types.ModuleType): # if this is already a module, we are done - mod = module - else: - # first try to import as a python path - try: - mod = __import__(module, fromlist=["None"]) - except ImportError as ex: - # check just where the ImportError happened (it could have been - # an erroneous import inside the module as well). This is the - # trivial way to do it ... - if not str(ex).startswith("No module named "): - raise + return module - # error in this module. Try absolute path import instead + if module.endswith('.py') and os.path.exists(module): + return mod_import_from_path(module) - if not os.path.isabs(module): - module = os.path.abspath(module) - path, filename = module.rsplit(os.path.sep, 1) - modname = re.sub(r"\.py$", "", filename) - - try: - result = imp.find_module(modname, [path]) - except ImportError: - logger.log_trace("Could not find module '%s' (%s.py) at path '%s'" % (modname, modname, path)) - return None - try: - mod = imp.load_module(modname, *result) - except ImportError: - logger.log_trace("Could not find or import module %s at path '%s'" % (modname, path)) - mod = None - # we have to close the file handle manually - result[0].close() - return mod + try: + return import_module(module) + except ImportError: + return None def all_from_module(module): From 901277ea646a4d5e13e6764f2a7da9cb55c51830 Mon Sep 17 00:00:00 2001 From: Greg Taylor Date: Sun, 15 Sep 2019 18:24:48 -0700 Subject: [PATCH 2/2] Switch to module imports for importlib in utils The present day guidance is to lean towards module imports for the stdlib modules. Switch importlib imports to this instead of plucking out the functions that we need. This makes it more immediately apparent as to where the functions are coming from in the application logic. --- evennia/utils/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/evennia/utils/utils.py b/evennia/utils/utils.py index ad2a500eab..64aa827aeb 100644 --- a/evennia/utils/utils.py +++ b/evennia/utils/utils.py @@ -16,12 +16,12 @@ import textwrap import random import inspect import traceback +import importlib +import importlib.util import importlib.machinery from twisted.internet.task import deferLater from twisted.internet.defer import returnValue # noqa - used as import target from os.path import join as osjoin -from importlib import import_module -from importlib.util import find_spec from inspect import ismodule, trace, getmembers, getmodule, getmro from collections import defaultdict, OrderedDict from twisted.internet import threads, reactor @@ -1216,7 +1216,7 @@ def mod_import(module): return mod_import_from_path(module) try: - return import_module(module) + return importlib.import_module(module) except ImportError: return None @@ -1378,7 +1378,7 @@ def fuzzy_import_from_module(path, variable, default=None, defaultpaths=None): paths = [path] + make_iter(defaultpaths) for modpath in paths: try: - mod = import_module(modpath) + mod = importlib.import_module(modpath) except ImportError as ex: if not str(ex).startswith("No module named %s" % modpath): # this means the module was found but it @@ -1420,13 +1420,13 @@ def class_from_module(path, defaultpaths=None): raise ImportError("the path '%s' is not on the form modulepath.Classname." % path) try: - if not find_spec(testpath, package='evennia'): + if not importlib.util.find_spec(testpath, package='evennia'): continue except ModuleNotFoundError: continue try: - mod = import_module(testpath, package='evennia') + mod = importlib.import_module(testpath, package='evennia') except ModuleNotFoundError: err = traceback.format_exc(30) break