diff --git a/evennia/utils/tests/test_utils.py b/evennia/utils/tests/test_utils.py index 8cde74f349..090126b67f 100644 --- a/evennia/utils/tests/test_utils.py +++ b/evennia/utils/tests/test_utils.py @@ -223,3 +223,20 @@ class TestImportFunctions(TestCase): test_path = self._t_dir_file("invalid_filename.py") loaded_mod = utils.mod_import_from_path(test_path) self.assertIsNone(loaded_mod) + + +class LatinifyTest(TestCase): + def setUp(self): + super().setUp() + + self.example_str = 'It naïvely says, “plugh.”' + self.expected_output = 'It naively says, "plugh."' + + def test_plain_string(self): + result = utils.latinify(self.example_str) + self.assertEqual(result, self.expected_output) + + def test_byte_string(self): + byte_str = utils.to_bytes(self.example_str) + result = utils.latinify(byte_str) + self.assertEqual(result, self.expected_output) diff --git a/evennia/utils/utils.py b/evennia/utils/utils.py index b39c1857ec..49c8af1e7b 100644 --- a/evennia/utils/utils.py +++ b/evennia/utils/utils.py @@ -761,7 +761,10 @@ _UNICODE_MAP = { "EN DASH": "-", "HORIZONTAL BAR": "-", "HORIZONTAL ELLIPSIS": "...", + "LEFT SINGLE QUOTATION MARK": "'", "RIGHT SINGLE QUOTATION MARK": "'", + "LEFT DOUBLE QUOTATION MARK": '"', + "RIGHT DOUBLE QUOTATION MARK": '"', } @@ -788,10 +791,13 @@ def latinify(string, default="?", pure_ascii=False): from unicodedata import name + if isinstance(string, bytes): + string = string.decode("utf8") + converted = [] for unich in iter(string): try: - ch = unich.decode("ascii") + ch = unich.encode("utf8").decode("ascii") except UnicodeDecodeError: # deduce a latin letter equivalent from the Unicode data # point name; e.g., since `name(u'á') == 'LATIN SMALL