From bdf6a9e72371bcea2ac4801cf06d855dc0a42e92 Mon Sep 17 00:00:00 2001 From: Wendy Wang Date: Tue, 15 Oct 2024 21:39:35 +0200 Subject: [PATCH 1/6] Database backup contrib with CmdBackup --- .../contrib/utils/database_backup/README.md | 69 +++++++ .../contrib/utils/database_backup/__init__.py | 5 + .../utils/database_backup/database_backup.py | 183 ++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 evennia/contrib/utils/database_backup/README.md create mode 100644 evennia/contrib/utils/database_backup/__init__.py create mode 100644 evennia/contrib/utils/database_backup/database_backup.py diff --git a/evennia/contrib/utils/database_backup/README.md b/evennia/contrib/utils/database_backup/README.md new file mode 100644 index 0000000000..e49f4c3069 --- /dev/null +++ b/evennia/contrib/utils/database_backup/README.md @@ -0,0 +1,69 @@ +# Database Backup Scheduler + +Contribution by helpme (2024) + +This module helps backup and restore your game world from database, as well as scheduling backups in-game. Backups are saved in your game's `server` folder. Database backups are *not* automatically uploaded to any cloud service, it is left to you to decide what to do with them (i.e. pushed to git, uploaded to a cloud service, downloaded to hard drive). + +Backups can take place at any time, while restoring the game world from backup has to take place outside of game commands, during downtime, as documented below. + +Currently, the sqlite3 (the evennia default) and postgresql databases are supported. Others are welcome to add more. + +## Installation + +This utility adds the `backup` command. Import the module into your commands and add it to your command set to make it available. + +In `mygame/commands/default_cmdsets.py`: + +```python +... +from evennia.contrib.utils.database_backup import DbCmdSet # <--- + +class CharacterCmdset(default_cmds.Character_CmdSet): + ... + def at_cmdset_creation(self): + ... + self.add(DbCmdSet) # <--- + +``` + +Then `reload` to make the `backup` command available. + +## Permissions + +By default, the backup command is only available to those with Developer permissions and higher. You can change this by overriding the command and setting its locks from "cmd:pperm(Developer)" to the lock of your choice. + +## Settings Used + +This utility uses the settings.DATABASES dictionary. + +## Restoration + +Remember to `evennia stop` before restoring your db. + +### Restoring sqlite3 (.db3) + +* Copy the database backup you want to restore from back into the `server/` directory +* Rename the database backup to `evennia.db3` if you have not modified `settings.DATABASES`, otherwise whatever name is in your `settings.DATABASES` dictionary. + +### Restoring postgres (.sql) + +* Prepare the following variables +``` +export DB_USER=db_user from your settings.DATABASES +export DB_NAME=db_name from your settings.DATABASES +export BACKUP_FILE=the path to the backup file you are restoring from +export PGPASSWORD=your db password + +If you prefer not to export your password to an env variable, you can enter it when prompted instead. +``` +* Run the following commands +``` +# Drop the existing database if it exists +psql -U $DB_USER -c "DROP DATABASE IF EXISTS $DB_NAME;" || exit 1 + +# Recreate the database +psql -U $DB_USER -c "CREATE DATABASE $DB_NAME;" || exit 1 + +# Restore the database from the backup file +psql -U $DB_USER -d $DB_NAME -f $BACKUP_FILE || exit 1 +``` \ No newline at end of file diff --git a/evennia/contrib/utils/database_backup/__init__.py b/evennia/contrib/utils/database_backup/__init__.py new file mode 100644 index 0000000000..f1b15f29fd --- /dev/null +++ b/evennia/contrib/utils/database_backup/__init__.py @@ -0,0 +1,5 @@ +""" +Database backups - helpme 2024 +""" + +from .database_backup import DbCmdSet # noqa diff --git a/evennia/contrib/utils/database_backup/database_backup.py b/evennia/contrib/utils/database_backup/database_backup.py new file mode 100644 index 0000000000..d1c6bdef1d --- /dev/null +++ b/evennia/contrib/utils/database_backup/database_backup.py @@ -0,0 +1,183 @@ +from django.conf import settings + +from evennia.comms.models import ChannelDB +from evennia import CmdSet, DefaultScript +from evennia.commands.default.muxcommand import MuxCommand +from evennia.utils.create import create_script +from evennia.utils import logger, search + +import datetime +import os +import shutil +import subprocess + +_MUDINFO_CHANNEL = None +BACKUP_FOLDER = "server/backups" +DATETIME_FORMAT_STR = "%Y-%m-%d.%H_%M_%S" +DEFAULT_INTERVAL = 86400 + + +class DatabaseBackupScript(DefaultScript): + """ + The global script to backup the server on a schedule. + + It will be automatically created the first time the `backup` command is used. + """ + + def at_script_creation(self): + super().at_script_creation() + self.key = "db_backup_script" + self.desc = "Database backups" + self.persistent = True + + def backup_postgres(self, db_name, db_user, output_file_path): + """ + Run `pg_dump` on the postgreSQL database and save the output. + + Returns: + str: Success message + """ + + output_file_path += ".sql" + subprocess.run( + ["pg_dump", "-U", db_user, "-F", "p", db_name], + stdout=open(output_file_path, "w"), + check=True, + ) + return f"|wPostgreSQL db backed up in: {BACKUP_FOLDER}|n" + + def backup_sqlite3(self, db_name, output_file_path): + """ + Copy the sqlite3 db. + + Returns: + str: Success message + """ + + output_file_path += ".db3" + os.makedirs(os.path.dirname(output_file_path), exist_ok=True) + shutil.copy(db_name, output_file_path) + return f"|wsqlite3 db backed up in {BACKUP_FOLDER}|n" + + def at_repeat(self): + global _MUDINFO_CHANNEL + if not _MUDINFO_CHANNEL: + if settings.CHANNEL_MUDINFO: + _MUDINFO_CHANNEL = ChannelDB.objects.get(db_key=settings.CHANNEL_MUDINFO["key"]) + + databases = settings.DATABASES + db = databases["default"] + engine = db.get("ENGINE") + db_name = db.get("NAME") + db_user = db.get("USER") + + try: + # Create the output folder if it doesn't exist + os.makedirs(BACKUP_FOLDER, exist_ok=True) + output_file = datetime.datetime.now().replace(microsecond=0).isoformat() + output_file_path = os.path.join(BACKUP_FOLDER, output_file) + + if "postgres" in engine: + message = self.backup_postgres(db_name, db_user, output_file_path) + elif "sqlite3" in engine: + message = self.backup_sqlite3(db_name, output_file_path) + + logger.log_sec(message) + if _MUDINFO_CHANNEL: + _MUDINFO_CHANNEL.msg(message) + except Exception as e: + logger.log_err("Backup failed: {}".format(e)) + + +class CmdBackup(MuxCommand): + """ + Backup your database to the server/backups folder in your game directory. + + Usage: + backup [interval in seconds] - Schedule a backup. The default interval is one day. + backup/stop - Stop the backup script (equivalent to scripts/delete #id) + backup/force - Trigger a backup manually + """ + + key = "backup" + aliases = "backups" + locks = "cmd:pperm(Developer)" + + def get_latest_backup(self): + try: + files = os.listdir(BACKUP_FOLDER) + paths = [os.path.join(BACKUP_FOLDER, basename) for basename in files] + last_backup = max(paths, key=os.path.getctime) + return last_backup + except Exception: + return "" + + def create_script(self, interval): + """Create new script. Deletes old script if it exists.""" + script = search.search_script("db_backup_script") + if script: + script[0].delete() + create_script(DatabaseBackupScript, interval=interval) + self.caller.msg(f"You have scheduled backups to run every {interval} seconds.") + + def get_script(self): + """ + Returns: + script: Existing script + """ + script = search.search_script("db_backup_script") + if script: + return script[0] + + def func(self): + """ + Database backup functionality + """ + caller = self.caller + args = self.args.strip() + interval = int(args) if args.isnumeric() else DEFAULT_INTERVAL + + script = self.get_script() + + # Kill existing backup script + if "stop" in self.switches: + if not script: + caller.msg("No existing db backup script to delete.") + return + script.delete() + caller.msg("DB backup script deleted.") + return + + # Create new backup script + if not script: + self.create_script(interval) + return + + # Change backup script's interval (if the provided interval is different) + original_interval = script.interval + if args and original_interval != interval: + self.create_script(interval) + return + + if "force" in self.switches: + # Manually trigger the backup + script.at_repeat() + return + + if self.get_latest_backup(): + caller.msg(f"Most recent database backup: {self.get_latest_backup()}.") + else: + caller.msg(f"No database backups found in {BACKUP_FOLDER}") + + caller.msg( + f"Countdown till next scheduled backup: |x{datetime.timedelta(seconds=script.time_until_next_repeat())}|n. Use |wbackup/force|n to manually backup the database." + ) + + +class DbCmdSet(CmdSet): + """ + Database backup command + """ + + def at_cmdset_creation(self): + self.add(CmdBackup) From 66d8126d195b8e25d496c5e7b7439e9eb9749306 Mon Sep 17 00:00:00 2001 From: Wendy Wang Date: Tue, 15 Oct 2024 21:47:17 +0200 Subject: [PATCH 2/6] Update docs --- evennia/contrib/utils/database_backup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evennia/contrib/utils/database_backup/README.md b/evennia/contrib/utils/database_backup/README.md index e49f4c3069..3e8e5a7882 100644 --- a/evennia/contrib/utils/database_backup/README.md +++ b/evennia/contrib/utils/database_backup/README.md @@ -2,9 +2,9 @@ Contribution by helpme (2024) -This module helps backup and restore your game world from database, as well as scheduling backups in-game. Backups are saved in your game's `server` folder. Database backups are *not* automatically uploaded to any cloud service, it is left to you to decide what to do with them (i.e. pushed to git, uploaded to a cloud service, downloaded to hard drive). +This module schedules backups in-game, which saves a copy of your database to your game's `server/backups` folder. Database backups are *not* automatically uploaded to any cloud service, it is left to you to decide what to do with them (i.e. pushed to git, uploaded to the ether, downloaded to a hard drive). -Backups can take place at any time, while restoring the game world from backup has to take place outside of game commands, during downtime, as documented below. +Backups can take place at any time. Restoring the game world from backup takes place during downtime, as documented below. Currently, the sqlite3 (the evennia default) and postgresql databases are supported. Others are welcome to add more. From 1c6ae9058f4aace3880719c2d6d5ca5b0285b5c2 Mon Sep 17 00:00:00 2001 From: Wendy Wang Date: Tue, 15 Oct 2024 23:02:50 +0200 Subject: [PATCH 3/6] Added tests, cleaned up messaging --- .../utils/database_backup/database_backup.py | 27 +++--- .../contrib/utils/database_backup/tests.py | 88 +++++++++++++++++++ 2 files changed, 103 insertions(+), 12 deletions(-) create mode 100644 evennia/contrib/utils/database_backup/tests.py diff --git a/evennia/contrib/utils/database_backup/database_backup.py b/evennia/contrib/utils/database_backup/database_backup.py index d1c6bdef1d..ff92837351 100644 --- a/evennia/contrib/utils/database_backup/database_backup.py +++ b/evennia/contrib/utils/database_backup/database_backup.py @@ -30,6 +30,17 @@ class DatabaseBackupScript(DefaultScript): self.desc = "Database backups" self.persistent = True + def log(self, message): + global _MUDINFO_CHANNEL + if not _MUDINFO_CHANNEL and settings.CHANNEL_MUDINFO: + channels = search.search_channel(settings.CHANNEL_MUDINFO["key"]) + if channels: + _MUDINFO_CHANNEL = channels[0] + + if _MUDINFO_CHANNEL: + _MUDINFO_CHANNEL.msg(message) + logger.log_sec(message) + def backup_postgres(self, db_name, db_user, output_file_path): """ Run `pg_dump` on the postgreSQL database and save the output. @@ -44,7 +55,7 @@ class DatabaseBackupScript(DefaultScript): stdout=open(output_file_path, "w"), check=True, ) - return f"|wPostgreSQL db backed up in: {BACKUP_FOLDER}|n" + self.log(f"|wpostgresql db backed up in: {BACKUP_FOLDER}|n") def backup_sqlite3(self, db_name, output_file_path): """ @@ -57,14 +68,9 @@ class DatabaseBackupScript(DefaultScript): output_file_path += ".db3" os.makedirs(os.path.dirname(output_file_path), exist_ok=True) shutil.copy(db_name, output_file_path) - return f"|wsqlite3 db backed up in {BACKUP_FOLDER}|n" + self.log(f"|wsqlite3 db backed up in: {BACKUP_FOLDER}|n") def at_repeat(self): - global _MUDINFO_CHANNEL - if not _MUDINFO_CHANNEL: - if settings.CHANNEL_MUDINFO: - _MUDINFO_CHANNEL = ChannelDB.objects.get(db_key=settings.CHANNEL_MUDINFO["key"]) - databases = settings.DATABASES db = databases["default"] engine = db.get("ENGINE") @@ -78,13 +84,10 @@ class DatabaseBackupScript(DefaultScript): output_file_path = os.path.join(BACKUP_FOLDER, output_file) if "postgres" in engine: - message = self.backup_postgres(db_name, db_user, output_file_path) + self.backup_postgres(db_name, db_user, output_file_path) elif "sqlite3" in engine: - message = self.backup_sqlite3(db_name, output_file_path) + self.backup_sqlite3(db_name, output_file_path) - logger.log_sec(message) - if _MUDINFO_CHANNEL: - _MUDINFO_CHANNEL.msg(message) except Exception as e: logger.log_err("Backup failed: {}".format(e)) diff --git a/evennia/contrib/utils/database_backup/tests.py b/evennia/contrib/utils/database_backup/tests.py new file mode 100644 index 0000000000..0c617de42f --- /dev/null +++ b/evennia/contrib/utils/database_backup/tests.py @@ -0,0 +1,88 @@ +""" +Tests for database backups. +""" + +import evennia.contrib.utils.database_backup.database_backup as backup +from evennia.commands.default.tests import BaseEvenniaCommandTest +from unittest.mock import patch + +EXCEPTION_STR = "failed" + + +class TestDatabaseBackupScript(BaseEvenniaCommandTest): + mocked_db_setting_postgres = patch( + "django.conf.settings.DATABASES", + { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": "fake_name", + "USER": "fake_user", + } + }, + ) + + def setUp(self): + super().setUp() + + @patch("shutil.copy") + @patch("evennia.utils.logger.log_sec") + def test_sqlite_success(self, mock_logger, mock_copy): + mock_copy.return_value.returncode = 0 + self.call( + backup.CmdBackup(), + "300", + "You have scheduled backups to run every 300 seconds.", + caller=self.char1, + ) + + mock_logger.assert_called_with(f"|wsqlite3 db backed up in: {backup.BACKUP_FOLDER}|n") + + self.call( + backup.CmdBackup(), + "/stop", + "DB backup script deleted.", + caller=self.char1, + ) + + @patch("shutil.copy") + @patch("evennia.utils.logger.log_err") + def test_sqlite_failure(self, mock_logger, mock_copy): + mock_copy.return_value.returncode = 1 + mock_copy.side_effect = Exception(EXCEPTION_STR) + + self.call( + backup.CmdBackup(), + "", + "You have scheduled backups to run every 86400 seconds.", + caller=self.char1, + ) + mock_logger.assert_called_with(f"Backup failed: {EXCEPTION_STR}") + + @mocked_db_setting_postgres + @patch("subprocess.run") + @patch("evennia.utils.logger.log_sec") + def test_postgres_success(self, mock_logger, mock_run): + mock_run.return_value.returncode = 0 + + self.call( + backup.CmdBackup(), + "", + "You have scheduled backups to run every 86400 seconds.", + caller=self.char1, + ) + mock_logger.assert_called_with(f"|wpostgresql db backed up in: {backup.BACKUP_FOLDER}|n") + + @mocked_db_setting_postgres + @patch("subprocess.run") + @patch("evennia.utils.logger.log_err") + def test_postgres_failure(self, mock_logger, mock_run): + mock_run.return_value.returncode = 0 + mock_run.side_effect = Exception(EXCEPTION_STR) + + self.call( + backup.CmdBackup(), + "", + "You have scheduled backups to run every 86400 seconds.", + caller=self.char1, + ) + mock_logger.assert_called_with(f"Backup failed: {EXCEPTION_STR}") From 41715a56c00840e12f7746faadefa147b64bd8fb Mon Sep 17 00:00:00 2001 From: Wendy Wang Date: Tue, 15 Oct 2024 23:13:43 +0200 Subject: [PATCH 4/6] Fix some docstrings --- .../contrib/utils/database_backup/database_backup.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/evennia/contrib/utils/database_backup/database_backup.py b/evennia/contrib/utils/database_backup/database_backup.py index ff92837351..8d725930b4 100644 --- a/evennia/contrib/utils/database_backup/database_backup.py +++ b/evennia/contrib/utils/database_backup/database_backup.py @@ -44,9 +44,6 @@ class DatabaseBackupScript(DefaultScript): def backup_postgres(self, db_name, db_user, output_file_path): """ Run `pg_dump` on the postgreSQL database and save the output. - - Returns: - str: Success message """ output_file_path += ".sql" @@ -60,9 +57,6 @@ class DatabaseBackupScript(DefaultScript): def backup_sqlite3(self, db_name, output_file_path): """ Copy the sqlite3 db. - - Returns: - str: Success message """ output_file_path += ".db3" @@ -107,6 +101,10 @@ class CmdBackup(MuxCommand): locks = "cmd:pperm(Developer)" def get_latest_backup(self): + """ + Returns: + str: Name of the most recent backup + """ try: files = os.listdir(BACKUP_FOLDER) paths = [os.path.join(BACKUP_FOLDER, basename) for basename in files] From b995e0340fb3a262cb194c43c896f6cd447eb8d3 Mon Sep 17 00:00:00 2001 From: Wendy Wang Date: Mon, 20 Jan 2025 17:06:49 +0100 Subject: [PATCH 5/6] Updated README per comments --- .../contrib/utils/database_backup/README.md | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/evennia/contrib/utils/database_backup/README.md b/evennia/contrib/utils/database_backup/README.md index 3e8e5a7882..be2122a355 100644 --- a/evennia/contrib/utils/database_backup/README.md +++ b/evennia/contrib/utils/database_backup/README.md @@ -10,7 +10,7 @@ Currently, the sqlite3 (the evennia default) and postgresql databases are suppor ## Installation -This utility adds the `backup` command. Import the module into your commands and add it to your command set to make it available. +This utility adds the `backup` command. The `backup` command can be used to set up a scheduled backup script, or trigger the script to run immediately. The backup script makes a backup of your game world. Import the module into your commands and add it to your command set to make it available. In `mygame/commands/default_cmdsets.py`: @@ -28,6 +28,21 @@ class CharacterCmdset(default_cmds.Character_CmdSet): Then `reload` to make the `backup` command available. +If you prefer to run the script without the `backup` command, you can manage it as a global script in your settings: + +```python +# in mygame/server/conf/settings.py + +GLOBAL_SCRIPTS = { + "backupscript": { + "typeclass": "evennia.contrib.utils.database_backup.DatabaseBackupScript", + "repeats": -1, + "interval": 86400, + "desc": "Database backup script" + }, +} +``` + ## Permissions By default, the backup command is only available to those with Developer permissions and higher. You can change this by overriding the command and setting its locks from "cmd:pperm(Developer)" to the lock of your choice. @@ -43,16 +58,16 @@ Remember to `evennia stop` before restoring your db. ### Restoring sqlite3 (.db3) * Copy the database backup you want to restore from back into the `server/` directory -* Rename the database backup to `evennia.db3` if you have not modified `settings.DATABASES`, otherwise whatever name is in your `settings.DATABASES` dictionary. +* By default (unless you changed the name of the file in `settings DATABASES`), the game data is expected to be located in the sqlite3 file `mygame/server/evennia.db3`. Copy your backup file over this file to recover your backup. ### Restoring postgres (.sql) * Prepare the following variables ``` -export DB_USER=db_user from your settings.DATABASES -export DB_NAME=db_name from your settings.DATABASES -export BACKUP_FILE=the path to the backup file you are restoring from -export PGPASSWORD=your db password +export DB_USER=db_user # db_user from your settings.DATABASES +export DB_NAME=db_name # db_name from your settings.DATABASES +export BACKUP_FILE=backup_file_path # the path to the backup file you are restoring from +export PGPASSWORD=db_password # the password to your db If you prefer not to export your password to an env variable, you can enter it when prompted instead. ``` From 99c38ced02981ded8a1a478c81544ca11166b8fc Mon Sep 17 00:00:00 2001 From: Wendy Wang Date: Mon, 20 Jan 2025 17:26:17 +0100 Subject: [PATCH 6/6] Add PRAGMA integrity_check post-copying sqlite3 db --- .../contrib/utils/database_backup/database_backup.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/evennia/contrib/utils/database_backup/database_backup.py b/evennia/contrib/utils/database_backup/database_backup.py index 8d725930b4..79e9e199fa 100644 --- a/evennia/contrib/utils/database_backup/database_backup.py +++ b/evennia/contrib/utils/database_backup/database_backup.py @@ -10,6 +10,7 @@ import datetime import os import shutil import subprocess +import sqlite3 _MUDINFO_CHANNEL = None BACKUP_FOLDER = "server/backups" @@ -62,6 +63,16 @@ class DatabaseBackupScript(DefaultScript): output_file_path += ".db3" os.makedirs(os.path.dirname(output_file_path), exist_ok=True) shutil.copy(db_name, output_file_path) + + # Check the integrity of the copied db + con = sqlite3.connect(output_file_path) + cur = con.cursor() + try: + cur.execute("PRAGMA integrity_check") + except sqlite3.DatabaseError: + self.log(f"|rsqlite3 db backup to {BACKUP_FOLDER} failed: integrity check failed|n") + con.close() + return self.log(f"|wsqlite3 db backed up in: {BACKUP_FOLDER}|n") def at_repeat(self):