From 0f3e0fadf7a978e876700a5483e61178f6e6f8cb Mon Sep 17 00:00:00 2001 From: Johnny Date: Fri, 21 Sep 2018 00:11:15 +0000 Subject: [PATCH 1/4] Reimplements Throttle as a standalone class with improved memory management. --- evennia/server/tests.py | 40 ++++++++++++++++ evennia/server/throttle.py | 98 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 evennia/server/throttle.py diff --git a/evennia/server/tests.py b/evennia/server/tests.py index e821d58583..352245e033 100644 --- a/evennia/server/tests.py +++ b/evennia/server/tests.py @@ -26,6 +26,9 @@ except ImportError: from django.test.runner import DiscoverRunner +from evennia.server.throttle import Throttle +from evennia.utils.test_resources import EvenniaTest + from .deprecations import check_errors @@ -77,3 +80,40 @@ class TestDeprecations(TestCase): self.assertRaises(DeprecationWarning, check_errors, MockSettings(setting)) # test check for WEBSERVER_PORTS having correct value self.assertRaises(DeprecationWarning, check_errors, MockSettings("WEBSERVER_PORTS", value=["not a tuple"])) + +class ThrottleTest(EvenniaTest): + """ + Class for testing the connection/IP throttle. + """ + def test_throttle(self): + ips = ('94.100.176.153', '45.56.148.77', '5.196.1.129') + kwargs = { + 'maxlim': 5, + 'timeout': 5 * 60 + } + + for ip in ips: + # Throttle should not be engaged by default + self.assertFalse(Throttle.check(ip, **kwargs)) + + # Pretend to fail a bunch of events + for x in xrange(5): + obj = Throttle.update(ip) + self.assertFalse(obj) + + # Next ones should be blocked + self.assertTrue(Throttle.check(ip, **kwargs)) + + for x in xrange(Throttle.cache_size * 2): + obj = Throttle.update(ip) + self.assertFalse(obj) + + # Should still be blocked + self.assertTrue(Throttle.check(ip, **kwargs)) + + # Number of values should be limited by cache size + self.assertEqual(Throttle.cache_size, len(Throttle.get(ip))) + + # There should only be (cache_size * num_ips) total in the Throttle cache + cache = Throttle.get() + self.assertEqual(sum([len(cache[x]) for x in cache.keys()]), Throttle.cache_size * len(ips)) \ No newline at end of file diff --git a/evennia/server/throttle.py b/evennia/server/throttle.py new file mode 100644 index 0000000000..670703f840 --- /dev/null +++ b/evennia/server/throttle.py @@ -0,0 +1,98 @@ +from collections import defaultdict, deque +import time + +_LATEST_FAILURES = defaultdict(deque) + +class Throttle(object): + """ + Keeps a running count of failed actions per IP address. + + Available methods indicate whether or not the number of failures exceeds a + particular threshold. + + This version of the throttle is usable by both the terminal server as well + as the web server, imposes limits on memory consumption by using deques + with length limits instead of open-ended lists, and removes sparse keys when + no recent failures have been recorded. + """ + + error_msg = 'Too many failed attempts; you must wait a few minutes before trying again.' + cache_size = 20 + + @classmethod + def get(cls, ip=None, storage=_LATEST_FAILURES): + """ + Convenience function that appends a new event to the table. + + Args: + ip (str, optional): IP address of requestor + + Returns: + storage (dict): When no IP is provided, returns a dict of all + current IPs being tracked and the timestamps of their recent + failures. + timestamps (deque): When an IP is provided, returns a deque of + timestamps of recent failures only for that IP. + + """ + if ip: return storage.get(ip, deque(maxlen=cls.cache_size)) + return storage + + @classmethod + def update(cls, ip): + """ + Convenience function that appends a new event to the table. + + Args: + ip (str): IP address of requestor + + Returns: + throttled (False): Always returns False + + """ + return cls.check(ip) + + @classmethod + def check(cls, ip, maxlim=None, timeout=None, storage=_LATEST_FAILURES): + """ + This will check the session's address against the + _LATEST_FAILURES dictionary to check they haven't + spammed too many fails recently. + + Args: + ip (str): IP address of requestor + maxlim (int): max number of attempts to allow + timeout (int): number of timeout seconds after + max number of tries has been reached. + + Returns: + throttled (bool): True if throttling is active, + False otherwise. + + Notes: + If maxlim and/or timeout are set, the function will + just do the comparison, not append a new datapoint. + """ + now = time.time() + ip = str(ip) + if maxlim and timeout: + # checking mode + latest_fails = storage[ip] + if latest_fails and len(latest_fails) >= maxlim: + # too many fails recently + if now - latest_fails[-1] < timeout: + # too soon - timeout in play + return True + else: + # timeout has passed. clear faillist + del(storage[ip]) + return False + else: + return False + else: + # store the time of the latest fail + if ip not in storage or not storage[ip].maxlen: + storage[ip] = deque(maxlen=cls.cache_size) + + storage[ip].append(time.time()) + return False \ No newline at end of file From 79a12a38d63ad65d4b7902a3681acdef6c52f838 Mon Sep 17 00:00:00 2001 From: Johnny Date: Fri, 21 Sep 2018 17:38:31 +0000 Subject: [PATCH 2/4] Further improvements; Throttle maintains its own storage and no longer requires it to be supplied as an arg. --- evennia/server/tests.py | 30 ++++++++------ evennia/server/throttle.py | 85 ++++++++++++++++++++------------------ 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/evennia/server/tests.py b/evennia/server/tests.py index 352245e033..94eac8206a 100644 --- a/evennia/server/tests.py +++ b/evennia/server/tests.py @@ -88,32 +88,38 @@ class ThrottleTest(EvenniaTest): def test_throttle(self): ips = ('94.100.176.153', '45.56.148.77', '5.196.1.129') kwargs = { - 'maxlim': 5, - 'timeout': 5 * 60 + 'limit': 5, + 'timeout': 15 * 60 } + throttle = Throttle(**kwargs) + for ip in ips: # Throttle should not be engaged by default - self.assertFalse(Throttle.check(ip, **kwargs)) + self.assertFalse(throttle.check(ip)) # Pretend to fail a bunch of events - for x in xrange(5): - obj = Throttle.update(ip) + for x in xrange(50): + obj = throttle.update(ip) self.assertFalse(obj) # Next ones should be blocked - self.assertTrue(Throttle.check(ip, **kwargs)) + self.assertTrue(throttle.check(ip)) - for x in xrange(Throttle.cache_size * 2): - obj = Throttle.update(ip) + for x in xrange(throttle.cache_size * 2): + obj = throttle.update(ip) self.assertFalse(obj) # Should still be blocked - self.assertTrue(Throttle.check(ip, **kwargs)) + self.assertTrue(throttle.check(ip)) # Number of values should be limited by cache size - self.assertEqual(Throttle.cache_size, len(Throttle.get(ip))) + self.assertEqual(throttle.cache_size, len(throttle.get(ip))) + + cache = throttle.get() + + # Make sure there are entries for each IP + self.assertEqual(len(ips), len(cache.keys())) # There should only be (cache_size * num_ips) total in the Throttle cache - cache = Throttle.get() - self.assertEqual(sum([len(cache[x]) for x in cache.keys()]), Throttle.cache_size * len(ips)) \ No newline at end of file + self.assertEqual(sum([len(cache[x]) for x in cache.keys()]), throttle.cache_size * len(ips)) \ No newline at end of file diff --git a/evennia/server/throttle.py b/evennia/server/throttle.py index 670703f840..56c88c63f2 100644 --- a/evennia/server/throttle.py +++ b/evennia/server/throttle.py @@ -1,8 +1,6 @@ from collections import defaultdict, deque import time -_LATEST_FAILURES = defaultdict(deque) - class Throttle(object): """ Keeps a running count of failed actions per IP address. @@ -17,12 +15,26 @@ class Throttle(object): """ error_msg = 'Too many failed attempts; you must wait a few minutes before trying again.' - cache_size = 20 - @classmethod - def get(cls, ip=None, storage=_LATEST_FAILURES): + def __init__(self, **kwargs): """ - Convenience function that appends a new event to the table. + Allows setting of throttle parameters. + + Kwargs: + limit (int): Max number of failures before imposing limiter + timeout (int): number of timeout seconds after + max number of tries has been reached. + cache_size (int): Max number of attempts to record per IP within a + rolling window; this is NOT the same as the limit after which + the throttle is imposed! + """ + self.storage = defaultdict(deque) + self.cache_size = self.limit = kwargs.get('limit', 5) + self.timeout = kwargs.get('timeout', 5 * 60) + + def get(self, ip=None): + """ + Convenience function that returns the storage table, or part of. Args: ip (str, optional): IP address of requestor @@ -35,64 +47,55 @@ class Throttle(object): timestamps of recent failures only for that IP. """ - if ip: return storage.get(ip, deque(maxlen=cls.cache_size)) - return storage + if ip: return self.storage.get(ip, deque(maxlen=self.cache_size)) + else: return self.storage - @classmethod - def update(cls, ip): + def update(self, ip): """ - Convenience function that appends a new event to the table. + Store the time of the latest failure/ Args: ip (str): IP address of requestor Returns: - throttled (False): Always returns False + None """ - return cls.check(ip) + # Enforce length limits + if not self.storage[ip].maxlen: + self.storage[ip] = deque(maxlen=self.cache_size) + + self.storage[ip].append(time.time()) - @classmethod - def check(cls, ip, maxlim=None, timeout=None, storage=_LATEST_FAILURES): + def check(self, ip): """ This will check the session's address against the - _LATEST_FAILURES dictionary to check they haven't - spammed too many fails recently. + storage dictionary to check they haven't spammed too many + fails recently. Args: ip (str): IP address of requestor - maxlim (int): max number of attempts to allow - timeout (int): number of timeout seconds after - max number of tries has been reached. Returns: throttled (bool): True if throttling is active, False otherwise. - Notes: - If maxlim and/or timeout are set, the function will - just do the comparison, not append a new datapoint. """ now = time.time() ip = str(ip) - if maxlim and timeout: - # checking mode - latest_fails = storage[ip] - if latest_fails and len(latest_fails) >= maxlim: - # too many fails recently - if now - latest_fails[-1] < timeout: - # too soon - timeout in play - return True - else: - # timeout has passed. clear faillist - del(storage[ip]) - return False + + # checking mode + latest_fails = self.storage[ip] + if latest_fails and len(latest_fails) >= self.limit: + # too many fails recently + if now - latest_fails[-1] < self.timeout: + # too soon - timeout in play + return True else: + # timeout has passed. clear faillist + del(self.storage[ip]) return False else: - # store the time of the latest fail - if ip not in storage or not storage[ip].maxlen: - storage[ip] = deque(maxlen=cls.cache_size) - - storage[ip].append(time.time()) - return False \ No newline at end of file + return False + + \ No newline at end of file From 2be80d0a12350e4e31ee47bdd8d722e49fbd32a8 Mon Sep 17 00:00:00 2001 From: Johnny Date: Fri, 21 Sep 2018 17:39:51 +0000 Subject: [PATCH 3/4] Integrates new Throttle with unconnected Commands; rate limits new account creation (partial fix for #1523). --- evennia/commands/default/unloggedin.py | 90 +++++++++----------------- 1 file changed, 31 insertions(+), 59 deletions(-) diff --git a/evennia/commands/default/unloggedin.py b/evennia/commands/default/unloggedin.py index bc7e69934f..f1b26137b7 100644 --- a/evennia/commands/default/unloggedin.py +++ b/evennia/commands/default/unloggedin.py @@ -4,13 +4,14 @@ Commands that are available from the connect screen. import re import time import datetime -from collections import defaultdict +from collections import defaultdict, deque from random import getrandbits from django.conf import settings from django.contrib.auth import authenticate from evennia.accounts.models import AccountDB from evennia.objects.models import ObjectDB from evennia.server.models import ServerConfig +from evennia.server.throttle import Throttle from evennia.comms.models import ChannelDB from evennia.server.sessionhandler import SESSIONS @@ -26,58 +27,11 @@ __all__ = ("CmdUnconnectedConnect", "CmdUnconnectedCreate", MULTISESSION_MODE = settings.MULTISESSION_MODE CONNECTION_SCREEN_MODULE = settings.CONNECTION_SCREEN_MODULE -# Helper function to throttle failed connection attempts. -# This can easily be used to limit account creation too, -# (just supply a different storage dictionary), but this -# would also block dummyrunner, so it's not added as default. - -_LATEST_FAILED_LOGINS = defaultdict(list) - - -def _throttle(session, maxlim=None, timeout=None, storage=_LATEST_FAILED_LOGINS): - """ - This will check the session's address against the - _LATEST_LOGINS dictionary to check they haven't - spammed too many fails recently. - - Args: - session (Session): Session failing - maxlim (int): max number of attempts to allow - timeout (int): number of timeout seconds after - max number of tries has been reached. - - Returns: - throttles (bool): True if throttling is active, - False otherwise. - - Notes: - If maxlim and/or timeout are set, the function will - just do the comparison, not append a new datapoint. - - """ - address = session.address - if isinstance(address, tuple): - address = address[0] - now = time.time() - if maxlim and timeout: - # checking mode - latest_fails = storage[address] - if latest_fails and len(latest_fails) >= maxlim: - # too many fails recently - if now - latest_fails[-1] < timeout: - # too soon - timeout in play - return True - else: - # timeout has passed. Reset faillist - storage[address] = [] - return False - else: - return False - else: - # store the time of the latest fail - storage[address].append(time.time()) - return False +# Create an object to store account creation attempts per IP +CREATION_THROTTLE = Throttle(limit=2, timeout=10 * 60) +# Create an object to store failed login attempts per IP +LOGIN_THROTTLE = Throttle(limit=5, timeout=5 * 60) def create_guest_account(session): """ @@ -134,7 +88,7 @@ def create_guest_account(session): session.msg("An error occurred. Please e-mail an admin if the problem persists.") logger.log_trace() raise - + def create_normal_account(session, name, password): """ @@ -149,8 +103,11 @@ def create_normal_account(session, name, password): account (Account): the account which was created from the name and password. """ # check for too many login errors too quick. - if _throttle(session, maxlim=5, timeout=5 * 60): - # timeout is 5 minutes. + address = session.address + if isinstance(address, tuple): + address = address[0] + + if LOGIN_THROTTLE.check(address): session.msg("|RYou made too many connection attempts. Try again in a few minutes.|n") return None @@ -161,7 +118,7 @@ def create_normal_account(session, name, password): # No accountname or password match session.msg("Incorrect login information given.") # this just updates the throttle - _throttle(session) + LOGIN_THROTTLE.update(address) # calls account hook for a failed login if possible. account = AccountDB.objects.get_account_from_name(name) if account: @@ -171,7 +128,6 @@ def create_normal_account(session, name, password): # Check IP and/or name bans bans = ServerConfig.objects.conf("server_bans") if bans and (any(tup[0] == account.name.lower() for tup in bans) or - any(tup[2].match(session.address) for tup in bans if tup[2])): # this is a banned IP or name! string = "|rYou have been banned and cannot continue from here." \ @@ -182,7 +138,6 @@ def create_normal_account(session, name, password): return account - class CmdUnconnectedConnect(COMMAND_DEFAULT_CLASS): """ connect to the game @@ -211,7 +166,10 @@ class CmdUnconnectedConnect(COMMAND_DEFAULT_CLASS): session = self.caller # check for too many login errors too quick. - if _throttle(session, maxlim=5, timeout=5 * 60, storage=_LATEST_FAILED_LOGINS): + address = session.address + if isinstance(address, tuple): + address = address[0] + if CONNECTION_THROTTLE.check(address): # timeout is 5 minutes. session.msg("|RYou made too many connection attempts. Try again in a few minutes.|n") return @@ -234,6 +192,7 @@ class CmdUnconnectedConnect(COMMAND_DEFAULT_CLASS): session.msg("\n\r Usage (without <>): connect ") return + CONNECTION_THROTTLE.update(address) name, password = parts account = create_normal_account(session, name, password) if account: @@ -262,6 +221,15 @@ class CmdUnconnectedCreate(COMMAND_DEFAULT_CLASS): session = self.caller args = self.args.strip() + + # Rate-limit account creation. + address = session.address + + if isinstance(address, tuple): + address = address[0] + if CREATION_THROTTLE.check(address): + session.msg("|RYou are creating too many accounts. Try again in a few minutes.|n") + return # extract double quoted parts parts = [part.strip() for part in re.split(r"\"", args) if part.strip()] @@ -322,6 +290,10 @@ class CmdUnconnectedCreate(COMMAND_DEFAULT_CLASS): if MULTISESSION_MODE < 2: default_home = ObjectDB.objects.get_id(settings.DEFAULT_HOME) _create_character(session, new_account, typeclass, default_home, permissions) + + # Update the throttle to indicate a new account was created from this IP + CREATION_THROTTLE.update(address) + # tell the caller everything went well. string = "A new account '%s' was created. Welcome!" if " " in accountname: From 8980f564ddf508579e0fcde2f225981baffe4b80 Mon Sep 17 00:00:00 2001 From: Johnny Date: Fri, 21 Sep 2018 17:45:59 +0000 Subject: [PATCH 4/4] Removes stray imports. --- evennia/commands/default/unloggedin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/evennia/commands/default/unloggedin.py b/evennia/commands/default/unloggedin.py index f1b26137b7..580272c420 100644 --- a/evennia/commands/default/unloggedin.py +++ b/evennia/commands/default/unloggedin.py @@ -4,7 +4,6 @@ Commands that are available from the connect screen. import re import time import datetime -from collections import defaultdict, deque from random import getrandbits from django.conf import settings from django.contrib.auth import authenticate