Add proper status reporting and stability fixes

This commit is contained in:
Griatch 2018-01-17 00:42:40 +01:00
parent 27afb3240d
commit 76f27f9bc2
6 changed files with 172 additions and 89 deletions

View file

@ -101,10 +101,11 @@ class AMPServerClientProtocol(amp.AMPMultiConnectionProtocol):
Called when a new connection is established.
"""
info_dict = self.factory.server.get_info_dict()
super(AMPServerClientProtocol, self).connectionMade()
# first thing we do is to request the Portal to sync all sessions
# back with the Server side
self.send_AdminServer2Portal(amp.DUMMYSESSION, operation=amp.PSYNC)
self.send_AdminServer2Portal(amp.DUMMYSESSION, operation=amp.PSYNC, info_dict=info_dict)
def data_to_portal(self, command, sessid, **kwargs):
"""

View file

@ -434,6 +434,66 @@ PROCESS_ERROR = \
{component} process error: {traceback}.
"""
PORTAL_INFO = \
"""{servername} Portal {version}
external ports:
{telnet}
{telnet_ssl}
{ssh}
{webserver_proxy}
{webclient}
internal_ports (to Server):
{amp}
{webserver_internal}
"""
SERVER_INFO = \
"""{servername} Server {version}
internal ports (to Portal):
{amp}
{webserver}
{irc_rss}
{info}
{errors}"""
# Info formatting
def print_info(portal_info_dict, server_info_dict):
"""
Format info dicts from the Portal/Server for display
"""
ind = " " * 7
def _prepare_dict(dct):
out = {}
for key, value in dct.iteritems():
if isinstance(value, list):
value = "\n{}".format(ind).join(value)
out[key] = value
return out
def _strip_empty_lines(string):
return "\n".join(line for line in string.split("\n") if line.strip())
pstr, sstr = "", ""
if portal_info_dict:
pdict = _prepare_dict(portal_info_dict)
pstr = _strip_empty_lines(PORTAL_INFO.format(**pdict))
if server_info_dict:
sdict = _prepare_dict(server_info_dict)
sstr = _strip_empty_lines(SERVER_INFO.format(**sdict))
info = pstr + ("\n\n" + sstr if sstr else "")
maxwidth = max(len(line) for line in info.split("\n"))
top_border = "-" * (maxwidth - 11) + " Evennia " + "--"
border = "-" * (maxwidth + 1)
print(top_border + "\n" + info + '\n' + border)
# ------------------------------------------------------------
#
# Protocol Evennia launcher - Portal/Server communication
@ -482,13 +542,17 @@ class AMPLauncherProtocol(amp.AMP):
@MsgStatus.responder
def receive_status_from_portal(self, status):
"""
Get a status signal from portal - fire callbacks
Get a status signal from portal - fire next queued
callback
"""
status = pickle.loads(status)
for callback in self.on_status:
try:
callback = self.on_status.pop()
except IndexError:
pass
else:
status = pickle.loads(status)
callback(status)
self.on_status = []
return {"status": ""}
@ -503,10 +567,6 @@ def send_instruction(operation, arguments, callback=None, errback=None):
print(ERROR_AMP_UNCONFIGURED)
sys.exit()
def _timeout(*args):
print("Client timed out.")
reactor.stop()
def _callback(result):
if callback:
callback(result)
@ -587,7 +647,7 @@ def _get_twistd_cmdline(pprofiler, sprofiler):
return portal_cmd, server_cmd
def query_status(repeat=False):
def query_status(callback=None):
"""
Send status ping to portal
@ -596,10 +656,13 @@ def query_status(repeat=False):
False: "NOT RUNNING"}
def _callback(response):
pstatus, sstatus, ppid, spid = _parse_status(response)
print("Portal: {} (pid {})\nServer: {} (pid {})".format(
wmap[pstatus], ppid, wmap[sstatus], spid))
reactor.stop()
if callback:
callback(response)
else:
pstatus, sstatus, ppid, spid, pinfo, sinfo = _parse_status(response)
print("Portal: {} (pid {})\nServer: {} (pid {})".format(
wmap[pstatus], ppid, wmap[sstatus], spid))
reactor.stop()
def _errback(fail):
pstatus, sstatus = False, False
@ -636,7 +699,7 @@ def wait_for_status(portal_running=True, server_running=True, callback=None, err
retries (int): How many times to retry before timing out and calling `errback`.
"""
def _callback(response):
prun, srun, _, _ = _parse_status(response)
prun, srun, _, _, _, _ = _parse_status(response)
if ((portal_running is None or prun == portal_running) and
(server_running is None or srun == server_running)):
# the correct state was achieved
@ -699,8 +762,11 @@ def start_evennia(pprofiler=False, sprofiler=False):
print(fail)
reactor.stop()
def _server_started(*args):
def _server_started(response):
print("... Server started.\nEvennia running.")
if response:
_, _, _, _, pinfo, sinfo = response
print_info(pinfo, sinfo)
reactor.stop()
def _portal_started(*args):
@ -708,7 +774,7 @@ def start_evennia(pprofiler=False, sprofiler=False):
send_instruction(SSTART, server_cmd)
def _portal_running(response):
prun, srun, ppid, spid = _parse_status(response)
prun, srun, ppid, spid, _, _ = _parse_status(response)
print("Portal is already running as process {pid}. Not restarted.".format(pid=ppid))
if srun:
print("Server is already running as process {pid}. Not restarted.".format(pid=spid))
@ -744,7 +810,7 @@ def reload_evennia(sprofiler=False, reset=False):
reactor.stop()
def _server_reloaded(status):
print("{} ... Server {}.".format(status, "reset" if reset else "reloaded"))
print("... Server {}.".format("reset" if reset else "reloaded"))
reactor.stop()
def _server_stopped(status):
@ -752,7 +818,7 @@ def reload_evennia(sprofiler=False, reset=False):
send_instruction(SSTART, server_cmd)
def _portal_running(response):
_, srun, _, _ = _parse_status(response)
_, srun, _, _, _, _ = _parse_status(response)
if srun:
print("Server {}...".format("resetting" if reset else "reloading"))
wait_for_status_reply(_server_stopped)
@ -785,7 +851,7 @@ def stop_evennia():
wait_for_status(False, None, _portal_stopped)
def _portal_running(response):
prun, srun, ppid, spid = _parse_status(response)
prun, srun, ppid, spid, _, _ = _parse_status(response)
if srun:
print("Server stopping ...")
send_instruction(SSHUTD, {})
@ -812,7 +878,7 @@ def stop_server_only():
reactor.stop()
def _portal_running(response):
_, srun, _, _ = _parse_status(response)
_, srun, _, _, _, _ = _parse_status(response)
if srun:
print("Server stopping ...")
wait_for_status_reply(_server_stopped)
@ -826,6 +892,25 @@ def stop_server_only():
send_instruction(PSTATUS, None, _portal_running, _portal_not_running)
def query_info():
"""
Display the info strings from the running Evennia
"""
def _got_status(status):
_, _, _, _, pinfo, sinfo = _parse_status(status)
print_info(pinfo, sinfo)
reactor.stop()
def _portal_running(response):
query_status(_got_status)
def _portal_not_running(fail):
print("Evennia is not running.")
send_instruction(PSTATUS, None, _portal_running, _portal_not_running)
def evennia_version():
"""
Get the Evennia version info from the main package.
@ -1735,12 +1820,14 @@ def main():
# launch menu for operation
init_game_directory(CURRENT_DIR, check_db=True)
run_menu()
elif option in ('status', 'sstart', 'sreload', 'sreset', 'sstop', 'ssstop', 'start', 'reload', 'stop'):
elif option in ('status', 'info', 'sstart', 'sreload', 'sreset', 'sstop', 'ssstop', 'start', 'reload', 'stop'):
# operate the server directly
init_game_directory(CURRENT_DIR, check_db=True)
if option == "status":
query_status()
if option == "sstart":
elif option == "info":
query_info()
elif option == "sstart":
start_evennia(False, args.profiler)
elif option == 'sreload':
reload_evennia(args.profiler)

View file

@ -302,7 +302,10 @@ class AMPMultiConnectionProtocol(amp.AMP):
portal will continuously try to reconnect, showing the problem
that way.
"""
self.factory.broadcasts.remove(self)
try:
self.factory.broadcasts.remove(self)
except ValueError:
pass
# Error handling

View file

@ -49,6 +49,7 @@ class AMPServerFactory(protocol.ServerFactory):
self.protocol = AMPServerProtocol
self.broadcasts = []
self.server_connection = None
self.server_info_dict = None
self.launcher_connection = None
self.disconnect_callbacks = {}
self.server_connect_callbacks = []
@ -83,6 +84,7 @@ class AMPServerProtocol(amp.AMPMultiConnectionProtocol):
super(AMPServerProtocol, self).connectionLost(reason)
if self.factory.server_connection == self:
self.factory.server_connection = None
self.factory.server_info_dict = None
if self.factory.launcher_connection == self:
self.factory.launcher_connection = None
@ -104,9 +106,11 @@ class AMPServerProtocol(amp.AMPMultiConnectionProtocol):
"""
server_connected = bool(self.factory.server_connection and
self.factory.server_connection.transport.connected)
portal_info_dict = self.factory.portal.get_info_dict()
server_info_dict = self.factory.server_info_dict
server_pid = self.factory.portal.server_process_id
portal_pid = os.getpid()
return (True, server_connected, portal_pid, server_pid)
return (True, server_connected, portal_pid, server_pid, portal_info_dict, server_info_dict)
def data_to_server(self, command, sessid, **kwargs):
"""
@ -276,10 +280,9 @@ class AMPServerProtocol(amp.AMPMultiConnectionProtocol):
"""
self.factory.launcher_connection = self
_, server_connected, _, _ = self.get_status()
_, server_connected, _, _, _, _ = self.get_status()
logger.log_msg("AMP SERVER operation == %s received" % (ord(operation)))
logger.log_msg("AMP SERVER arguments: %s" % (amp.loads(arguments)))
logger.log_msg("Evennia Launcher->Portal operation %s received" % (ord(operation)))
if operation == amp.SSTART: # portal start #15
# first, check if server is already running
@ -395,13 +398,13 @@ class AMPServerProtocol(amp.AMPMultiConnectionProtocol):
elif operation == amp.PSYNC: # portal sync
# Server has (re-)connected and wants the session data from portal
self.factory.server_info_dict = kwargs.get("info_dict", {})
sessdata = self.factory.portal.sessions.get_all_sync_data()
self.send_AdminPortal2Server(amp.DUMMYSESSION,
amp.PSYNC,
sessiondata=sessdata)
self.factory.portal.sessions.at_server_connection()
print("Portal PSYNC: %s" % self.factory.server_connection)
if self.factory.server_connection:
# this is an indication the server has successfully connected, so
# we trigger any callbacks (usually to tell the launcher server is up)

View file

@ -7,7 +7,6 @@ sets up all the networking features. (this is done automatically
by game/evennia.py).
"""
from __future__ import print_function
from builtins import object
import sys
@ -77,9 +76,13 @@ AMP_PORT = settings.AMP_PORT
AMP_INTERFACE = settings.AMP_INTERFACE
AMP_ENABLED = AMP_HOST and AMP_PORT and AMP_INTERFACE
INFO_DICT = {"servername": SERVERNAME, "version": VERSION, "errors": "", "info": "",
"lockdown_mode": "", "amp": "", "telnet": [], "telnet_ssl": [], "ssh": [],
"webclient": [], "webserver_proxy": [], "webserver_internal": []}
# -------------------------------------------------------------
# Portal Service object
# -------------------------------------------------------------
class Portal(object):
@ -114,6 +117,10 @@ class Portal(object):
self.game_running = False
def get_info_dict(self):
"Return the Portal info, for display."
return INFO_DICT
def set_restart_mode(self, mode=None):
"""
This manages the flag file that tells the runner if the server
@ -127,7 +134,6 @@ class Portal(object):
if mode is None:
return
with open(PORTAL_RESTART, 'w') as f:
print("writing mode=%(mode)s to %(portal_restart)s" % {'mode': mode, 'portal_restart': PORTAL_RESTART})
f.write(str(mode))
def shutdown(self, restart=None, _reactor_stopping=False):
@ -148,7 +154,6 @@ class Portal(object):
case it always needs to be restarted manually.
"""
print("portal.shutdown: restart=", restart)
if _reactor_stopping and hasattr(self, "shutdown_complete"):
# we get here due to us calling reactor.stop below. No need
# to do the shutdown procedure again.
@ -179,10 +184,9 @@ application = service.Application('Portal')
# and is where we store all the other services.
PORTAL = Portal(application)
print('-' * 50)
print(' %(servername)s Portal (%(version)s) started.' % {'servername': SERVERNAME, 'version': VERSION})
if LOCKDOWN_MODE:
print(' LOCKDOWN_MODE active: Only local connections.')
INFO_DICT["lockdown_mode"] = ' LOCKDOWN_MODE active: Only local connections.'
if AMP_ENABLED:
@ -192,7 +196,7 @@ if AMP_ENABLED:
from evennia.server.portal import amp_server
print(' amp (to Server): %s (internal)' % AMP_PORT)
INFO_DICT["amp"] = 'amp: %s)' % AMP_PORT
factory = amp_server.AMPServerFactory(PORTAL)
amp_service = internet.TCPServer(AMP_PORT, factory, interface=AMP_INTERFACE)
@ -223,12 +227,12 @@ if TELNET_ENABLED:
telnet_service.setName('EvenniaTelnet%s' % pstring)
PORTAL.services.addService(telnet_service)
print(' telnet%s: %s (external)' % (ifacestr, port))
INFO_DICT["telnet"].append('telnet%s: %s' % (ifacestr, port))
if SSL_ENABLED:
# Start SSL game connection (requires PyOpenSSL).
# Start Telnet+SSL game connection (requires PyOpenSSL).
from evennia.server.portal import ssl
@ -249,7 +253,7 @@ if SSL_ENABLED:
ssl_service.setName('EvenniaSSL%s' % pstring)
PORTAL.services.addService(ssl_service)
print(" ssl%s: %s (external)" % (ifacestr, port))
INFO_DICT["telnet_ssl"].append("telnet+ssl%s: %s" % (ifacestr, port))
if SSH_ENABLED:
@ -273,7 +277,7 @@ if SSH_ENABLED:
ssh_service.setName('EvenniaSSH%s' % pstring)
PORTAL.services.addService(ssh_service)
print(" ssh%s: %s (external)" % (ifacestr, port))
INFO_DICT["ssh"].append("ssh%s: %s" % (ifacestr, port))
if WEBSERVER_ENABLED:
@ -296,7 +300,7 @@ if WEBSERVER_ENABLED:
ajax_webclient = webclient_ajax.AjaxWebClient()
ajax_webclient.sessionhandler = PORTAL_SESSIONS
web_root.putChild("webclientdata", ajax_webclient)
webclientstr = "\n + webclient (ajax only)"
webclientstr = "webclient (ajax only)"
if WEBSOCKET_CLIENT_ENABLED and not websocket_started:
# start websocket client port for the webclient
@ -314,10 +318,11 @@ if WEBSERVER_ENABLED:
factory.protocol = webclient.WebSocketClient
factory.sessionhandler = PORTAL_SESSIONS
websocket_service = internet.TCPServer(port, WebSocketFactory(factory), interface=w_interface)
websocket_service.setName('EvenniaWebSocket%s:%s' % (w_ifacestr, proxyport))
websocket_service.setName('EvenniaWebSocket%s:%s' % (w_ifacestr, port))
PORTAL.services.addService(websocket_service)
websocket_started = True
webclientstr = "\n + webclient-websocket%s: %s (external)" % (w_ifacestr, proxyport)
webclientstr = "webclient-websocket%s: %s" % (w_ifacestr, port)
INFO_DICT["webclient"].append(webclientstr)
web_root = Website(web_root, logPath=settings.HTTP_LOG_FILE)
proxy_service = internet.TCPServer(proxyport,
@ -325,16 +330,10 @@ if WEBSERVER_ENABLED:
interface=interface)
proxy_service.setName('EvenniaWebProxy%s' % pstring)
PORTAL.services.addService(proxy_service)
print(" website-proxy%s: %s (external) %s" % (ifacestr, proxyport, webclientstr))
INFO_DICT["webserver_proxy"].append("website%s: %s" % (ifacestr, proxyport))
INFO_DICT["webserver_internal"].append("webserver: %s" % serverport)
for plugin_module in PORTAL_SERVICES_PLUGIN_MODULES:
# external plugin services to start
plugin_module.start_plugin_services(PORTAL)
print('-' * 50) # end of terminal output
if os.name == 'nt':
# Windows only: Set PID file manually
with open(PORTAL_PIDFILE, 'w') as f:
f.write(str(os.getpid()))

View file

@ -7,7 +7,6 @@ sets up all the networking features. (this is done automatically
by evennia/server/server_runner.py).
"""
from __future__ import print_function
from builtins import object
import time
import sys
@ -40,11 +39,6 @@ from django.utils.translation import ugettext as _
_SA = object.__setattr__
SERVER_PIDFILE = ""
if os.name == 'nt':
# For Windows we need to handle pid files manually.
SERVER_PIDFILE = os.path.join(settings.GAME_DIR, "server", 'server.pid')
# a file with a flag telling the server to restart after shutdown or not.
SERVER_RESTART = os.path.join(settings.GAME_DIR, "server", 'server.restart')
@ -53,12 +47,7 @@ SERVER_STARTSTOP_MODULE = mod_import(settings.AT_SERVER_STARTSTOP_MODULE)
# modules containing plugin services
SERVER_SERVICES_PLUGIN_MODULES = [mod_import(module) for module in make_iter(settings.SERVER_SERVICES_PLUGIN_MODULES)]
try:
WEB_PLUGINS_MODULE = mod_import(settings.WEB_PLUGINS_MODULE)
except ImportError:
WEB_PLUGINS_MODULE = None
print ("WARNING: settings.WEB_PLUGINS_MODULE not found - "
"copy 'evennia/game_template/server/conf/web_plugins.py to mygame/server/conf.")
#------------------------------------------------------------
# Evennia Server settings
@ -83,6 +72,17 @@ IRC_ENABLED = settings.IRC_ENABLED
RSS_ENABLED = settings.RSS_ENABLED
WEBCLIENT_ENABLED = settings.WEBCLIENT_ENABLED
INFO_DICT = {"servername": SERVERNAME, "version": VERSION,
"amp": "", "errors": "", "info": "", "webserver": "", "irc_rss": ""}
try:
WEB_PLUGINS_MODULE = mod_import(settings.WEB_PLUGINS_MODULE)
except ImportError:
WEB_PLUGINS_MODULE = None
INFO_DICT["errors"] = (
"WARNING: settings.WEB_PLUGINS_MODULE not found - "
"copy 'evennia/game_template/server/conf/web_plugins.py to mygame/server/conf.")
# Maintenance function - this is called repeatedly by the server
@ -231,6 +231,8 @@ class Evennia(object):
typeclasses in the settings file and have them auto-update all
already existing objects.
"""
global INFO_DICT
# setting names
settings_names = ("CMDSET_CHARACTER", "CMDSET_ACCOUNT",
"BASE_ACCOUNT_TYPECLASS", "BASE_OBJECT_TYPECLASS",
@ -249,7 +251,7 @@ class Evennia(object):
#from evennia.accounts.models import AccountDB
for i, prev, curr in ((i, tup[0], tup[1]) for i, tup in enumerate(settings_compare) if i in mismatches):
# update the database
print(" %s:\n '%s' changed to '%s'. Updating unchanged entries in database ..." % (settings_names[i], prev, curr))
INFO_DICT['info'] = " %s:\n '%s' changed to '%s'. Updating unchanged entries in database ..." % (settings_names[i], prev, curr)
if i == 0:
ObjectDB.objects.filter(db_cmdset_storage__exact=prev).update(db_cmdset_storage=curr)
if i == 1:
@ -279,29 +281,27 @@ class Evennia(object):
It returns if this is not the first time the server starts.
Once finished the last_initial_setup_step is set to -1.
"""
global INFO_DICT
last_initial_setup_step = ServerConfig.objects.conf('last_initial_setup_step')
if not last_initial_setup_step:
# None is only returned if the config does not exist,
# i.e. this is an empty DB that needs populating.
print(' Server started for the first time. Setting defaults.')
INFO_DICT['info'] = ' Server started for the first time. Setting defaults.'
initial_setup.handle_setup(0)
print('-' * 50)
elif int(last_initial_setup_step) >= 0:
# a positive value means the setup crashed on one of its
# modules and setup will resume from this step, retrying
# the last failed module. When all are finished, the step
# is set to -1 to show it does not need to be run again.
print(' Resuming initial setup from step %(last)s.' %
{'last': last_initial_setup_step})
INFO_DICT['info'] = ' Resuming initial setup from step {last}.'.format(
last=last_initial_setup_step)
initial_setup.handle_setup(int(last_initial_setup_step))
print('-' * 50)
def run_init_hooks(self):
"""
Called every server start
"""
from evennia.objects.models import ObjectDB
#from evennia.accounts.models import AccountDB
# update eventual changed defaults
self.update_defaults()
@ -366,7 +366,6 @@ class Evennia(object):
once - in both cases the reactor is
dead/stopping already.
"""
print("server.shutdown mode=", mode)
if _reactor_stopping and hasattr(self, "shutdown_complete"):
# this means we have already passed through this method
# once; we don't need to run the shutdown procedure again.
@ -414,10 +413,6 @@ class Evennia(object):
# always called, also for a reload
self.at_server_stop()
if os.name == 'nt' and os.path.exists(SERVER_PIDFILE):
# for Windows we need to remove pid files manually
os.remove(SERVER_PIDFILE)
if hasattr(self, "web_root"): # not set very first start
yield self.web_root.empty_threadpool()
@ -429,6 +424,10 @@ class Evennia(object):
# we make sure the proper gametime is saved as late as possible
ServerConfig.objects.conf("runtime", _GAMETIME_MODULE.runtime())
def get_info_dict(self):
"Return the server info, for display."
return INFO_DICT
# server start/stop hooks
def at_server_start(self):
@ -536,9 +535,6 @@ application = service.Application('Evennia')
# and is where we store all the other services.
EVENNIA = Evennia(application)
print('-' * 50)
print(' %(servername)s Server (%(version)s) started.' % {'servername': SERVERNAME, 'version': VERSION})
if AMP_ENABLED:
# The AMP protocol handles the communication between
@ -548,7 +544,8 @@ if AMP_ENABLED:
ifacestr = ""
if AMP_INTERFACE != '127.0.0.1':
ifacestr = "-%s" % AMP_INTERFACE
print(' amp (to Portal)%s: %s (internal)' % (ifacestr, AMP_PORT))
INFO_DICT["amp"] = 'amp %s: %s' % (ifacestr, AMP_PORT)
from evennia.server import amp_client
@ -561,7 +558,6 @@ if WEBSERVER_ENABLED:
# Start a django-compatible webserver.
#from twisted.python import threadpool
from evennia.server.webserver import DjangoWebRoot, WSGIWebServer, Website, LockableThreadPool
# start a thread pool and define the root url (/) as a wsgi resource
@ -582,13 +578,14 @@ if WEBSERVER_ENABLED:
web_site = Website(web_root, logPath=settings.HTTP_LOG_FILE)
INFO_DICT["webserver"] = ""
for proxyport, serverport in WEBSERVER_PORTS:
# create the webserver (we only need the port for this)
webserver = WSGIWebServer(threads, serverport, web_site, interface='127.0.0.1')
webserver.setName('EvenniaWebServer%s' % serverport)
EVENNIA.services.addService(webserver)
print(" webserver: %s (internal)" % serverport)
INFO_DICT["webserver"] += "webserver: %s" % serverport
ENABLED = []
if IRC_ENABLED:
@ -600,18 +597,11 @@ if RSS_ENABLED:
ENABLED.append('rss')
if ENABLED:
print(" " + ", ".join(ENABLED) + " enabled.")
INFO_DICT["irc_rss"] = ", ".join(ENABLED) + " enabled."
for plugin_module in SERVER_SERVICES_PLUGIN_MODULES:
# external plugin protocols
plugin_module.start_plugin_services(EVENNIA)
print('-' * 50) # end of terminal output
# clear server startup mode
ServerConfig.objects.conf("server_starting_mode", delete=True)
if os.name == 'nt':
# Windows only: Set PID file manually
with open(SERVER_PIDFILE, 'w') as f:
f.write(str(os.getpid()))