From edb0f70e2e78a3fdb3a316830452cc3848d7a92f Mon Sep 17 00:00:00 2001 From: Mathew Guest Date: Wed, 15 Jan 2020 17:56:00 -0700 Subject: [PATCH] first python imlementation, 80% of script converted --- README.md | 0 edit-config.sh | 0 setup.py | 42 +++++ smileyface.py | 4 + smileyface/__init__.py | 38 ++++ smileyface/app.py | 137 ++++++++++++++ smileyface/cmd_menu.py | 0 smileyface/config.spec | 24 +++ smileyface/model.py | 404 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 649 insertions(+) create mode 100644 README.md create mode 100644 edit-config.sh create mode 100755 setup.py create mode 100755 smileyface.py create mode 100644 smileyface/__init__.py create mode 100644 smileyface/app.py create mode 100644 smileyface/cmd_menu.py create mode 100644 smileyface/config.spec create mode 100644 smileyface/model.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/edit-config.sh b/edit-config.sh new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..12ff834 --- /dev/null +++ b/setup.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# +# Usage: +# +# First, enable the python environment you want to install to, or if installing +# system-wide then ensure you're logged in with sufficient permissions +# (admin or root to install to system directories) +# +# installation: +# +# $ ./setup.py install +# +# de-installation: +# +# $ pip uninstall + + +from setuptools import setup + +__project__ = 'SmileyFace UT4 Server Panel' +__version__ = '0.1.0' + +setup( + name = __project__, + version = __version__, + description = 'Unreal Tournament 4 Server Admin and Control Panel', + author = 'Mathew Guest', + author_email = 't3h.zavage@gmail.com', + url = 'https://git-mirror.zavage-software.com', + + # Third-party dependencies; will be automatically installed + install_requires = ( + 'app_skellington' + ), + + # Local packages to be installed (our packages) + packages = ( + 'smileyface', + ), + +) + diff --git a/smileyface.py b/smileyface.py new file mode 100755 index 0000000..2aa1b6d --- /dev/null +++ b/smileyface.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +import smileyface +smileyface.start_app() + diff --git a/smileyface/__init__.py b/smileyface/__init__.py new file mode 100644 index 0000000..471b833 --- /dev/null +++ b/smileyface/__init__.py @@ -0,0 +1,38 @@ +import logging +import sys + +# Module parameters and constants +APP_NAME = 'SmileyFace Unreal Tournament 4 Server Panel' +APP_AUTHOR = 'Mathew Guest' +APP_VERSION = '0.1.0' + +APP_CONFIG_FILENAME = 'config.ini' + +# config.spec is relative to the module src directory and is the +# config specification (structure, names, and types of config file) +APP_CONFIGSPEC_FILENAME = 'config.spec' + +# Check and gracefully fail if the user needs to install a 3rd-party dep. +required_lib_names = ['appdirs', 'configobj', 'colorlog'] +def check_env_has_dependencies(required_lib_names): + """ + Attempts to import each module and gracefully fails if it doesn't + exist. + """ + rc = True + for libname in required_lib_names: + try: + __import__(libname) + except ImportError as ex: + print('missing third-part library: ', ex, file=sys.stderr) + rc = False + except Exception as ex: + print(ex, type(ex)) + rc = False + return rc +if not check_env_has_dependencies(required_lib_names): + print('refusing to load program without installed dependencies', file=sys.stderr) + raise ImportError('python environment needs third-party dependencies installed') + +# Exposed from sub-modules: +from .app import start_app diff --git a/smileyface/app.py b/smileyface/app.py new file mode 100644 index 0000000..a70af95 --- /dev/null +++ b/smileyface/app.py @@ -0,0 +1,137 @@ +import app_skellington +from app_skellington import _util + +from . import model + +class SmileyFace(app_skellington.ApplicationContainer): + def __init__(self, *args, **kwargs): + filename = 'config.spec' + self.configspec_filepath = _util.get_asset(__name__, filename) + + super().__init__( + configspec_filepath=self.configspec_filepath, + app_name = 'SmileyFace UT4 Server Panel', + app_author = 'Mathew Guest', + app_version = '0.1', + *args, + **kwargs + ) + # super().__init__( + # app_name = 'SmileyFace UT4 Server Panel', + # app_author = 'Mathew Guest', + # app_version = '0.1' + # ) + self._load_config() + + def _load_config(self, config_file=None): + """ + Parse the config file, (todo) environment variables, and command-line + arguments to determine runtime configuration. + """ + if config_file is None: + config_file = self._get_config_filepath( + 'app_name', + 'app_author', + 'app_config_filename' + ) + + rc = self.ctx.config.load_config_from_file(config_file) + + + + def _cli_options(self): + pass + + def _command_menu(self): + sm_root = self.cli.init_submenu('command') + # sm_root.register_command(model.foo) + # sm_root.register_command(model.bar) + _util.register_class_as_commands( + self, sm_root, + model.UT4ServerMachine + ) + + def _services(self): + self['model'] = lambda: model.UTServerMachine(self.ctx) + + def interactive_shell(self): + pass + + def invoke_from_cli(self): + rc = self.load_command() + if not rc: + print('Invalid command. Try -h for usage') + return + # load config + self.invoke_command() + + def usage(self): + s = """ +Unreal Tournament 4 Server Build and Deploy Script + +A list of commands is shown below. + +List commands and usage: + ./ut4-server-ctl.sh + +Show help for a specific command: + ./ut4-server-ctl.sh --help + +Here is the list of sub-commands with short syntax reminder: + ./ut4-server-ctl.sh 1click-deploy + ./ut4-server-ctl.sh clean-instance + ./ut4-server-ctl.sh create-directories + ./ut4-server-ctl.sh download-linux-server + ./ut4-server-ctl.sh download-logs + ./ut4-server-ctl.sh generate-instance + ./ut4-server-ctl.sh start-server + ./ut4-server-ctl.sh stop-server + ./ut4-server-ctl.sh upload-redirects + ./ut4-server-ctl.sh upload-server + +Typical Usage: + 1) You need to either configure the remote server hostnames (both game server and remote redirect server) + in the vars files. Edit vars with a text editor, or edit the defaults in this file, or manually type + them interactively when prompted* (coming soon). + + e.g.: + PROJECT_DIR="/path/to/this/script/directory/on/local/machine" + REMOTE_GAME_HOST="54.123.456.10" + REMOTE_GAME_DIR="/home/ut4/hub-instance" + + REMOTE_REDIRECT_HOST="45.321.654.10" + REMOTE_REDIRECT_DIR="/srv/ut4-redirect/" + + 2) You need to download the latest Linux Server release from Epic. Do + this with the 'download-server' command. + + e.g.: + ./ut4-server-ctl.sh download-server + + 3) Add and configure custom maps, mutators, rulesets, and hub configuration to your *local* project folder. + This is done by modifying the files in the project directory. With the current environment variables, + this is set to be: + + "$PROJECT_DIR" + + This is the fun part! Connect with UTCC or UTZONE.DE (unaffiliated) to find custom content to put on + your hub. + + 4) 1click-deploy to update the remote hub and redirect with your latest content. You're done! Rinse and repeat. + + e.g.: + ./ut4-server-ctl.sh 1click-deploy + + Alternatively, this command is equivalent to running the separate commands. If you'd like more fine-grained + control, you can run them individually. + ./ut4-server-ctl.sh generate-instance + ./ut4-server-ctl.sh upload-redirects + ./ut4-server-ctl.sh upload-server + ./ut4-server-ctl.sh restart-server +""" + print(s) + +def start_app(): + app = SmileyFace() + app.invoke_from_cli() + diff --git a/smileyface/cmd_menu.py b/smileyface/cmd_menu.py new file mode 100644 index 0000000..e69de29 diff --git a/smileyface/config.spec b/smileyface/config.spec new file mode 100644 index 0000000..4f66721 --- /dev/null +++ b/smileyface/config.spec @@ -0,0 +1,24 @@ +[app] + project_dir = string(max=255, default='') + + config_dir = string(max=255, default='') + + download_url = string(max=255, default='https://s3.amazonaws.com/unrealtournament/ShippedBuilds/%2B%2BUT%2BRelease-Next-CL-3525360/UnrealTournament-Server-XAN-3525360-Linux.zip') + + download_filename = string(max=255, default='UnrealTournament-Server-XAN-3525360-Linux.zip') + + download_md5 = string(max=255, default='cad730ad6793ba6261f9a341ad7396eb') + + skip_validate = boolean(default=False) + + redirect_protocol = string(max=255, default='') + + redirect_url = string(max=255, default='') + + remote_game_host = string(max=255, default='') + + remote_game_directory = string(max=255, default='') + + remote_redirect_host = string(max=255, default='') + +[logging] diff --git a/smileyface/model.py b/smileyface/model.py new file mode 100644 index 0000000..f0fc674 --- /dev/null +++ b/smileyface/model.py @@ -0,0 +1,404 @@ +from app_skellington import _util +from functools import partial +import collections +import configparser +import glob +import hashlib +import os +import subprocess +import pathlib +import configobj +import sys + +class UT4ServerMachine: + def __init__(self, ctx): + self.ctx = ctx + + if not self._validate_env_vars(): + sys.exit(1) + + def oneclickdeploy(self): + self.generate_instance() + self.upload_redirects() + self.upload_server() + + def clean_instance(self, x): + """ + Deletes the generated instance on the local machine. + """ + self.ctx.log['ut4'].info('Clearing .pak folder...') + cmd = 'rm -rv "$PROJECT_DIR"/instance/LinuxServer/UnrealTournament/Content/Paks/*' + self._invoke_command(cmd) + + def create_directories(self): + """ + Create required directories which the user installs maps, mutators, and config to. + """ + dirs = ( + 'base', + 'files/config', + 'files/maps', + 'files/mutators', + 'files/rulesets', + 'files/unused' + ) + project_dir = self.ctx.config['app']['project_dir'] + if len(project_dir.strip()) == 0: + project_dir = '.' + print('project_dir:', project_dir) + fullpaths = [ + '/'.join([project_dir, d]) for d in dirs + ] + + for fp in fullpaths: + cmd = 'mkdir -p {}'.format(fp) + self._invoke_command(cmd) + + def download_linux_server(self, x): + """ + Download the latest Linux Unreal Tournament 4 Server from Epic + """ + self.ctx.log['ut4'].info('Downloading Linux Server Binary from Epic.') + + def download_logs(self): + """ + Download the logs from the target hub. + """ + config_dir = self.ctx.config['app']['config_dir'] + remote_game_host = self.ctx.config['app']['remote_game_host'] + remote_game_dir = self.ctx.config['app']['remote_game_dir'] + + self.ctx.log['ut4'].info('Downloading instance logs from target hub.') + cmd = ''' +rsync -ravzp {remote_game_host}:{remote_game_dir}/LinuxServer/UnrealTournament/Saved/Logs/ {config_dir}/downloaded-logs/ +'''\ +.format(**{ + 'config_dir': config_dir, + 'remote_game_host': remote_game_host, + 'remote_game_dir': remote_game_dir +}) + self._invoke_command(cmd) + + + # Delete logs on remote game server if successfully transferred to local: + self.ctx.log['ut4'].info('') + cmd = ''' +ssh {remote_game_host} rm {remote_game_dir}/LinuxServer/UnrealTournament/Saved/Logs/* -r' +'''\ +.format(**{ + 'config_dir': config_dir, + 'remote_game_host': remote_game_host, + 'remote_game_dir': remote_game_dir +}) + # self._invoke_command(cmd) + + def generate_instance(self): + """ + Takes the current coniguration and outputs the application files which + can be copied to the server. + """ + self.ctx.log['ut4'].info('Generating server instance from custom files...') + project_dir = self.ctx.config['app']['project_dir'] + + # rsync + src = '/'.join([project_dir, 'base/LinuxServer']) + dst = '/'.join([project_dir, 'instance/']) + cmd = 'rsync -ravzp {src} {dst}'.format(**{ + 'src': src, + 'dst': dst + }) + self._invoke_command(cmd) + + # cp 1 + src = '/'.join([project_dir, 'start-server.sh']) + dst = '/'.join([project_dir, 'instance/']) + cmd = 'cp {src} {dst}'.format(**{ + 'src': src, + 'dst': dst + }) + self._invoke_command(cmd) + + # cp 2 + src = '/'.join([project_dir, 'stop-server.sh']) + dst = '/'.join([project_dir, 'instance/']) + cmd = 'cp {src} {dst}'.format(**{ + 'src': src, + 'dst': dst + }) + self._invoke_command(cmd) + + # self._first_run() + self._install_config() + self._install_paks() + self._install_redirect_lines() + self._install_rulesets() + + def restart_server(self): + self.stop_server() + self.start_server() + + def start_server(self): + """ + Flip on the target hub on for Fragging! + """ + self.ctx.log['ut4'].info('Starting hub...') + + cmd = ''' +ssh {remote_game_host} {remote_game_dir}/start-server.sh +''' + self._invoke_command(cmd) + + def stop_server(self): + """ + Stop UT4 Hub processes on the server. + """ + self.ctx.log['ut4'].info('Stopping hub.') + cmd = ''' +ssh {remote_game_host} {remote_game_dir}/stop-server.sh +''' + self._invoke_command(cmd) + + def upload_redirects(self): + """ + Upload paks to redirect server. + """ + self.ctx.log['ut4'].info('Uploading redirects (maps, mutators, etc.) to target hub.') + pass + + def upload_server(self): + """ + Upload all required game files to the hub server. + """ + self.ctx.log['ut4'].info('Uploading customized server') + project_dir = self.ctx.config['app']['project_dir'] + remote_game_host = self.ctx.config['app']['remote_game_host'] + remote_game_dir = self.ctx.config['app']['remote_game_dir'] + cwd = None + + # transfer #1 + cmd = ''' +rsync -ravzp \ + --delete \ + --exclude ".KEEP" \ + --exclude "Mods.db" \ + --exclude "Mod.ini" \ + --exclude "Logs" \ + --exclude "ut4-server.log" \ + --exclude "Saved/*.ai" \ + --exclude "Saved/Crashes/*" \ + --exclude "Saved/Logs/*" \ + {project_dir}/instance/ \ + {remote_game_host}:{remote_game_dir}" +'''\ +.format(**{ + 'project_dir': project_dir, + 'remote_game_host': remote_game_host, + 'remote_game_dir': remote_game_dir +}) + subprocess.run(cmd, cwd=cwd) # should be invoke_command? + + # transfer #2 + cmd = ''' +rsync -avzp {project_dir}/ut4-server-ctl.sh {remote_game_host}:{remote_game_dir} +''' + subprocess.run(cmd, cwd=cwd) + + # transfer #3 + cmd = ''' +scp {project_dir}/instance/ut4-server.service {remote_game_host}:/etc/systemd/system/ +''' + subprocess.run(cmd, cwd=cwd) + + # transfer #4 + cmd = ''' +ssh {remote_game_host} chown ut4.ut4 {remote_game_dir} -R +''' + subprocess.run(cmd, cwd=cwd) + + def _first_run(self): + self.ctx.log['ut4'].info('Starting instance once to get UID.') + self.ctx.log['ut4'].info('Unfortunately, this takes 20 seconds. Just wait.') + + # cd "$PROJECT_DIR"/instance/LinuxServer/Engine/Binaries/Linux + # chmod 770 UE4Server-Linux-Shipping + # ./UE4Server-Linux-Shipping UnrealTournament UT-Entry?Game=Lobby -log &>/dev/null & + # cd - >/dev/null + + self.ctx.log['ut4'].info('sleeping 20 seconds and then we\'ll kill the server we started just now.') + # sleep 20 + # stop_server + # # TODO(MG) get uid and export + + def _install_config(self): + files = ( + 'Game.ini', + 'Engine.ini' + ) + project_dir = self.ctx.config['app']['project_dir'] + config_dir = self.ctx.config['app']['config_dir'] + for fn in files: + self.ctx.log['ut4'].info('Installing file: %s', fn) + # src = '/'.join([config_dir, fn]) + # dst = '/'.join([project_dir, 'instance']) # needs corrected + cmd = 'cp {src} {dst}'.format(**{ + 'src': '/'.join([config_dir, fn]), + 'dst': '/'.join([project_dir, 'instance']) # needs corrected + }) + self._invoke_command(cmd) + + def _install_paks(self): + project_dir = self.ctx.config['app']['project_dir'] + + self.ctx.log['ut4'].info('Installing maps...') + cmd = 'rsync -ravzp {src} {dst}'.format(**{ + 'src': '/'.join([project_dir, 'files/maps']), + 'dst': '/'.join([project_dir, 'instance/LinuxServer/UnrealTournament/Content/Paks/']) + }) + self._invoke_command(cmd) + + self.ctx.log['ut4'].info('Installing mutators...') + cmd = 'rsync -ravzp {src} {dst}'.format(**{ + 'src': '/'.join([project_dir, 'files/mutators']), + 'dst': '/'.join([project_dir, 'instance/LinuxServer/UnrealTournament/Content/Paks/']) + }) + self._invoke_command(cmd) + + def _install_redirect_lines(self): + return self.install_redirect_lines() + + def install_redirect_lines(self): + self.ctx.log['ut4'].info('Generating redirect references...') + redirect_protocol = self.ctx.config['app']['redirect_protocol'] + redirect_url = self.ctx.config['app']['redirect_url'] + project_dir = self.ctx.config['app']['project_dir'] + mod_dir = '/'.join([project_dir, 'files']) + files = glob.iglob('{}/**/*.pak'.format(mod_dir)) + redirect_lines = [] + for idx, filename in enumerate(files): + if idx > 5: + break + md5sum = self._md5sum_file(filename) + p = pathlib.Path(filename) + relative_path = p.relative_to(mod_dir) + basename = p.name + + line = '\ +RedirectReferences=(PackageName="{basename}",\ +PackageURLProtocol="{redirect_protocol}",\ +PackageURL="{redirect_url}/{relative_path}",\ +PackageChecksum="{md5sum}")'\ +.format(**{ + 'basename': basename, + 'redirect_protocol': redirect_protocol, + 'redirect_url': redirect_url, + 'relative_path': relative_path, + 'md5sum': md5sum +}) + self.ctx.log['ut4'].debug("redirect line = '%s'", line) + redirect_lines.append(line) + # end loop - for filename in files: + + # START HERE --trying to dynamically add redirect references and + # gonna need to be able to work with the ini file, including support + # for duplicate keys. Alternatively, maybe do the redirect lines through + # a patch + + # I found out the hard way, configobj doesn't seem the best suited for + # duplicate keys... + + # install into Game.ini + game_ini_filepath = '/'.join([project_dir, 'files/config/Game.ini']) + # game_ini_data = configobj.ConfigObj(game_ini_filepath) + config = configparser.ConfigParser( + # game_ini_filepath, + # dict_type=MultiOrderedDict, + strict=False + ) + s = config.sections() + print(s) + # config.set('/Script/UnrealTournament.UTBaseGameMode', 'RedirectReferences', line) + config.read(game_ini_filepath) + l = config.write(sys.stdout) + print('l', l) + + + def _install_rulesets(self): + self.ctx.log['ut4'].info('Concatenating rulesets for game modes...') + project_dir = self.ctx.config['app']['project_dir'] + + src_dir = '/'.join([project_dir, 'files/rulesets']) + out_dir = '/'.join([project_dir, '/instance/LinuxServer/UnrealTournament/Saved/Rulesets']) + out_filename='/'.join([out_dir, 'ruleset.json']) + + cmd = 'mkdir -pv {out_dir}'.format(**{ + 'out_dir': out_dir + }) + self._invoke_command(cmd) + + self.ctx.log['ut4'].info('out filename=%s', out_filename) + + # echo {\"rules\":[ > "$OUT_FILENAME" + cmd = 'echo {\"rules\":[ > "{out_filename}"' + self._invoke_command(cmd) + + cmd = 'for f in "{src_dir}"/*.json ; do ; cat "$f" >> "{out_filename}" ; done'.format( + **{ + 'src_dir': src_dir, + 'out_filename': out_filename + }) + self._invoke_command(cmd) + + cmd = 'echo "]}}" >> "{out_filename}"'.format( + **{ + 'out_filename': out_filename + }) + self._invoke_command(cmd) + self.ctx.log['ut4'].info('output ruleset is at "%s"', out_filename) + + def _invoke_command(self, cmd, msg=None): + assert isinstance(cmd, str), 'cmd input must be string: %s'.format(cmd) + if msg is None: + msg = cmd + print(msg) + self.ctx.log['ut4'].info('running cmd: %s', cmd) + cwd = None # todo(mg) ? + # os.system(cmd) + + def _md5sum_file(self, filename): + with open(filename, mode='rb') as f: + d = hashlib.md5() + for buf in iter(partial(f.read, 128), b''): + d.update(buf) + h = d.hexdigest() + return h + + def _validate_env_vars(self): + variable_names = ( + 'project_dir', + 'download_url', + 'download_filename', + 'download_md5', + 'redirect_protocol', + 'redirect_url', + 'remote_game_host', + 'remote_game_directory', + 'remote_redirect_host' + ) + for name in variable_names: + value = self.ctx.config['app'][name] + self.ctx.log['ut4'].info('%s: %s', name, value) + + i = input('Continue with above configuration? (y/N):') + if i.lower() != 'y': + self.ctx.log['ut4'].info('Doing nothing.') + return False + self.ctx.log['ut4'].info('Continuing.') + return True + +class MultiOrderedDict(collections.OrderedDict): + def __setitem__(self, key, value): + if isinstance(value, list) and key in self: + self[key].extend(value) + else: + super().__setitem__(key, value) +