first python imlementation, 80% of script converted

setup.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+#
+# Usage:
+# 
+#   First, enable the python environment you want to install to, or if installing
+#   system-wide then ensure you're logged in with sufficient permissions
+#   (admin or root to install to system directories)
+#
+#   installation:
+#
+#       $ ./setup.py install
+#
+#   de-installation:
+#
+#       $ pip uninstall <app>
+
+
+from setuptools import setup
+
+__project__ = 'SmileyFace UT4 Server Panel'
+__version__ = '0.1.0'
+
+setup(
+    name = __project__,
+    version = __version__,
+    description = 'Unreal Tournament 4 Server Admin and Control Panel',
+    author = 'Mathew Guest',
+    author_email = 't3h.zavage@gmail.com',
+    url = 'https://git-mirror.zavage-software.com',
+
+    # Third-party dependencies; will be automatically installed
+    install_requires = (
+        'app_skellington'
+    ),
+
+    # Local packages to be installed (our packages)
+    packages = (
+        'smileyface',
+    ),
+
+)
+

smileyface.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+import smileyface
+smileyface.start_app()
+

smileyface/__init__.py

@@ -0,0 +1,38 @@
+import logging
+import sys
+
+# Module parameters and constants
+APP_NAME = 'SmileyFace Unreal Tournament 4 Server Panel'
+APP_AUTHOR = 'Mathew Guest'
+APP_VERSION = '0.1.0'
+
+APP_CONFIG_FILENAME = 'config.ini'
+
+# config.spec is relative to the module src directory and is the
+# config specification (structure, names, and types of config file)
+APP_CONFIGSPEC_FILENAME = 'config.spec'
+
+# Check and gracefully fail if the user needs to install a 3rd-party dep.
+required_lib_names = ['appdirs', 'configobj', 'colorlog']
+def check_env_has_dependencies(required_lib_names):
+    """
+    Attempts to import each module and gracefully fails if it doesn't
+    exist.
+    """
+    rc = True
+    for libname in required_lib_names:
+        try:
+            __import__(libname)
+        except ImportError as ex:
+            print('missing third-part library: ', ex, file=sys.stderr)
+            rc = False
+        except Exception as ex:
+            print(ex, type(ex))
+            rc = False
+    return rc
+if not check_env_has_dependencies(required_lib_names):
+    print('refusing to load program without installed dependencies', file=sys.stderr)
+    raise ImportError('python environment needs third-party dependencies installed')
+
+# Exposed from sub-modules:
+from .app import start_app

smileyface/app.py

@@ -0,0 +1,137 @@
+import app_skellington
+from app_skellington import _util
+
+from . import model
+
+class SmileyFace(app_skellington.ApplicationContainer):
+    def __init__(self, *args, **kwargs):
+        filename = 'config.spec'
+        self.configspec_filepath = _util.get_asset(__name__, filename)
+
+        super().__init__(
+            configspec_filepath=self.configspec_filepath,
+            app_name = 'SmileyFace UT4 Server Panel',
+            app_author = 'Mathew Guest',
+            app_version = '0.1',
+            *args,
+            **kwargs
+        )
+        # super().__init__(
+        #     app_name = 'SmileyFace UT4 Server Panel',
+        #     app_author = 'Mathew Guest',
+        #     app_version = '0.1'
+        # )
+        self._load_config()
+
+    def _load_config(self, config_file=None):
+        """
+        Parse the config file, (todo) environment variables, and command-line
+        arguments to determine runtime configuration.
+        """
+        if config_file is None:
+            config_file = self._get_config_filepath(
+                'app_name',
+                'app_author',
+                'app_config_filename'
+            )
+
+        rc = self.ctx.config.load_config_from_file(config_file)
+
+
+
+    def _cli_options(self):
+        pass
+
+    def _command_menu(self):
+        sm_root = self.cli.init_submenu('command')
+        # sm_root.register_command(model.foo)
+        # sm_root.register_command(model.bar)
+        _util.register_class_as_commands(
+            self, sm_root,
+            model.UT4ServerMachine
+        )
+
+    def _services(self):
+        self['model'] = lambda: model.UTServerMachine(self.ctx)
+
+    def interactive_shell(self):
+        pass
+
+    def invoke_from_cli(self):
+        rc = self.load_command()
+        if not rc:
+            print('Invalid command. Try -h for usage')
+            return
+        # load config
+        self.invoke_command()
+
+    def usage(self):
+        s = """
+Unreal Tournament 4 Server Build and Deploy Script
+
+A list of commands is shown below. 
+
+List commands and usage:
+        ./ut4-server-ctl.sh 
+
+Show help for a specific command:
+        ./ut4-server-ctl.sh --help <COMMAND>
+
+Here is the list of sub-commands with short syntax reminder:
+        ./ut4-server-ctl.sh 1click-deploy
+        ./ut4-server-ctl.sh clean-instance
+        ./ut4-server-ctl.sh create-directories
+        ./ut4-server-ctl.sh download-linux-server
+        ./ut4-server-ctl.sh download-logs
+        ./ut4-server-ctl.sh generate-instance
+        ./ut4-server-ctl.sh start-server
+        ./ut4-server-ctl.sh stop-server
+        ./ut4-server-ctl.sh upload-redirects
+        ./ut4-server-ctl.sh upload-server
+
+Typical Usage:
+        1) You need to either configure the remote server hostnames (both game server and remote redirect server)
+        in the vars files. Edit vars with a text editor, or edit the defaults in this file, or manually type
+        them interactively when prompted* (coming soon).
+
+        e.g.:
+                PROJECT_DIR="/path/to/this/script/directory/on/local/machine"
+                REMOTE_GAME_HOST="54.123.456.10"
+                REMOTE_GAME_DIR="/home/ut4/hub-instance"
+
+                REMOTE_REDIRECT_HOST="45.321.654.10"
+                REMOTE_REDIRECT_DIR="/srv/ut4-redirect/"
+
+        2) You need to download the latest Linux Server release from Epic. Do
+        this with the 'download-server' command.
+
+        e.g.:
+                ./ut4-server-ctl.sh download-server
+
+        3) Add and configure custom maps, mutators, rulesets, and hub configuration to your *local* project folder.
+        This is done by modifying the files in the project directory. With the current environment variables,
+        this is set to be:
+        
+                "$PROJECT_DIR"
+
+        This is the fun part! Connect with UTCC or UTZONE.DE (unaffiliated) to find custom content to put on
+        your hub.
+
+        4) 1click-deploy to update the remote hub and redirect with your latest content. You're done! Rinse and repeat.
+
+        e.g.:
+                ./ut4-server-ctl.sh 1click-deploy
+
+        Alternatively, this command is equivalent to running the separate commands. If you'd like more fine-grained
+        control, you can run them individually.
+                ./ut4-server-ctl.sh generate-instance
+                ./ut4-server-ctl.sh upload-redirects
+                ./ut4-server-ctl.sh upload-server
+                ./ut4-server-ctl.sh restart-server
+"""
+        print(s) 
+
+def start_app():
+    app = SmileyFace()
+    app.invoke_from_cli()
+

smileyface/config.spec

@@ -0,0 +1,24 @@
+[app]
+        project_dir = string(max=255, default='')
+
+        config_dir = string(max=255, default='')
+
+        download_url = string(max=255, default='https://s3.amazonaws.com/unrealtournament/ShippedBuilds/%2B%2BUT%2BRelease-Next-CL-3525360/UnrealTournament-Server-XAN-3525360-Linux.zip')
+
+        download_filename = string(max=255, default='UnrealTournament-Server-XAN-3525360-Linux.zip')
+
+        download_md5 = string(max=255, default='cad730ad6793ba6261f9a341ad7396eb')
+
+        skip_validate = boolean(default=False)
+
+        redirect_protocol = string(max=255, default='')
+
+        redirect_url = string(max=255, default='')
+        
+        remote_game_host = string(max=255, default='')
+
+        remote_game_directory = string(max=255, default='')
+
+        remote_redirect_host = string(max=255, default='')
+
+[logging]

smileyface/model.py

@@ -0,0 +1,404 @@
+from app_skellington import _util
+from functools import partial
+import collections
+import configparser
+import glob
+import hashlib
+import os
+import subprocess
+import pathlib
+import configobj
+import sys
+
+class UT4ServerMachine:
+    def __init__(self, ctx):
+        self.ctx = ctx
+
+        if not self._validate_env_vars():
+            sys.exit(1)
+
+    def oneclickdeploy(self):
+        self.generate_instance()
+        self.upload_redirects()
+        self.upload_server()
+
+    def clean_instance(self, x):
+        """
+        Deletes the generated instance on the local machine.
+        """
+        self.ctx.log['ut4'].info('Clearing .pak folder...')
+        cmd = 'rm -rv "$PROJECT_DIR"/instance/LinuxServer/UnrealTournament/Content/Paks/*'
+        self._invoke_command(cmd)
+
+    def create_directories(self):
+        """
+        Create required directories which the user installs maps, mutators, and config to.
+        """
+        dirs = (
+            'base',
+            'files/config',
+            'files/maps',
+            'files/mutators',
+            'files/rulesets',
+            'files/unused'
+        )
+        project_dir = self.ctx.config['app']['project_dir']
+        if len(project_dir.strip()) == 0:
+            project_dir = '.'
+        print('project_dir:', project_dir)
+        fullpaths = [
+            '/'.join([project_dir, d]) for d in dirs
+        ]
+
+        for fp in fullpaths:
+            cmd = 'mkdir -p {}'.format(fp)
+            self._invoke_command(cmd)
+
+    def download_linux_server(self, x):
+        """
+        Download the latest Linux Unreal Tournament 4 Server from Epic
+        """
+        self.ctx.log['ut4'].info('Downloading Linux Server Binary from Epic.')
+
+    def download_logs(self):
+        """
+        Download the logs from the target hub.
+        """
+        config_dir = self.ctx.config['app']['config_dir']
+        remote_game_host = self.ctx.config['app']['remote_game_host']
+        remote_game_dir = self.ctx.config['app']['remote_game_dir']
+
+        self.ctx.log['ut4'].info('Downloading instance logs from target hub.')
+        cmd = '''
+rsync -ravzp {remote_game_host}:{remote_game_dir}/LinuxServer/UnrealTournament/Saved/Logs/ {config_dir}/downloaded-logs/
+'''\
+.format(**{
+    'config_dir': config_dir,
+    'remote_game_host': remote_game_host,
+    'remote_game_dir': remote_game_dir
+})
+        self._invoke_command(cmd)
+
+
+        # Delete logs on remote game server if successfully transferred to local:
+        self.ctx.log['ut4'].info('')
+        cmd = '''
+ssh {remote_game_host} rm {remote_game_dir}/LinuxServer/UnrealTournament/Saved/Logs/* -r'
+'''\
+.format(**{
+    'config_dir': config_dir,
+    'remote_game_host': remote_game_host,
+    'remote_game_dir': remote_game_dir
+})
+        # self._invoke_command(cmd)
+
+    def generate_instance(self):
+        """
+        Takes the current coniguration and outputs the application files which
+        can be copied to the server.
+        """
+        self.ctx.log['ut4'].info('Generating server instance from custom files...')
+        project_dir = self.ctx.config['app']['project_dir']
+        
+        # rsync
+        src = '/'.join([project_dir, 'base/LinuxServer'])
+        dst = '/'.join([project_dir, 'instance/'])
+        cmd = 'rsync -ravzp {src} {dst}'.format(**{
+            'src': src,
+            'dst': dst
+        })
+        self._invoke_command(cmd)
+
+        # cp 1
+        src = '/'.join([project_dir, 'start-server.sh'])
+        dst = '/'.join([project_dir, 'instance/'])
+        cmd = 'cp {src} {dst}'.format(**{
+            'src': src,
+            'dst': dst
+        })
+        self._invoke_command(cmd)
+
+        # cp 2
+        src = '/'.join([project_dir, 'stop-server.sh'])
+        dst = '/'.join([project_dir, 'instance/'])
+        cmd = 'cp {src} {dst}'.format(**{
+            'src': src,
+            'dst': dst
+        })
+        self._invoke_command(cmd)
+
+        # self._first_run()
+        self._install_config()
+        self._install_paks()
+        self._install_redirect_lines()
+        self._install_rulesets()
+
+    def restart_server(self):
+        self.stop_server()
+        self.start_server()
+
+    def start_server(self):
+        """
+        Flip on the target hub on for Fragging!
+        """
+        self.ctx.log['ut4'].info('Starting hub...')
+
+        cmd = '''
+ssh {remote_game_host} {remote_game_dir}/start-server.sh
+'''
+        self._invoke_command(cmd)
+
+    def stop_server(self):
+        """
+        Stop UT4 Hub processes on the server.
+        """
+        self.ctx.log['ut4'].info('Stopping hub.')
+        cmd = '''
+ssh {remote_game_host} {remote_game_dir}/stop-server.sh
+'''
+        self._invoke_command(cmd)
+
+    def upload_redirects(self):
+        """
+        Upload paks to redirect server.
+        """
+        self.ctx.log['ut4'].info('Uploading redirects (maps, mutators, etc.) to target hub.')
+        pass
+
+    def upload_server(self):
+        """
+        Upload all required game files to the hub server.
+        """
+        self.ctx.log['ut4'].info('Uploading customized server')
+        project_dir = self.ctx.config['app']['project_dir']
+        remote_game_host = self.ctx.config['app']['remote_game_host']
+        remote_game_dir = self.ctx.config['app']['remote_game_dir']
+        cwd = None
+
+        # transfer #1
+        cmd = '''
+rsync -ravzp \
+        --delete \
+        --exclude ".KEEP" \
+        --exclude "Mods.db" \
+        --exclude "Mod.ini" \
+        --exclude "Logs" \
+        --exclude "ut4-server.log" \
+        --exclude "Saved/*.ai" \
+        --exclude "Saved/Crashes/*" \
+        --exclude "Saved/Logs/*" \
+        {project_dir}/instance/ \
+        {remote_game_host}:{remote_game_dir}"
+'''\
+.format(**{
+    'project_dir': project_dir,
+    'remote_game_host': remote_game_host,
+    'remote_game_dir': remote_game_dir
+})
+        subprocess.run(cmd, cwd=cwd) # should be invoke_command?
+
+        # transfer #2
+        cmd = '''
+rsync -avzp {project_dir}/ut4-server-ctl.sh {remote_game_host}:{remote_game_dir}
+'''
+        subprocess.run(cmd, cwd=cwd)
+
+        # transfer #3
+        cmd = '''
+scp {project_dir}/instance/ut4-server.service {remote_game_host}:/etc/systemd/system/
+'''
+        subprocess.run(cmd, cwd=cwd)
+
+        # transfer #4
+        cmd = '''
+ssh {remote_game_host} chown ut4.ut4 {remote_game_dir} -R
+'''
+        subprocess.run(cmd, cwd=cwd)
+
+    def _first_run(self):
+        self.ctx.log['ut4'].info('Starting instance once to get UID.')
+        self.ctx.log['ut4'].info('Unfortunately, this takes 20 seconds. Just wait.')
+
+        # cd "$PROJECT_DIR"/instance/LinuxServer/Engine/Binaries/Linux
+        # chmod 770 UE4Server-Linux-Shipping
+        # ./UE4Server-Linux-Shipping UnrealTournament UT-Entry?Game=Lobby -log &>/dev/null &
+        # cd - >/dev/null
+
+        self.ctx.log['ut4'].info('sleeping 20 seconds and then we\'ll kill the server we started just now.')
+        # sleep 20
+        # stop_server
+        # # TODO(MG) get uid and export
+
+    def _install_config(self):
+        files = (
+            'Game.ini',
+            'Engine.ini'
+        )
+        project_dir = self.ctx.config['app']['project_dir']
+        config_dir = self.ctx.config['app']['config_dir']
+        for fn in files:
+            self.ctx.log['ut4'].info('Installing file: %s', fn)
+            # src = '/'.join([config_dir, fn])
+            # dst = '/'.join([project_dir, 'instance']) # needs corrected
+            cmd = 'cp {src} {dst}'.format(**{
+                'src': '/'.join([config_dir, fn]),
+                'dst': '/'.join([project_dir, 'instance']) # needs corrected
+            })
+            self._invoke_command(cmd)
+
+    def _install_paks(self):
+        project_dir = self.ctx.config['app']['project_dir']
+
+        self.ctx.log['ut4'].info('Installing maps...')
+        cmd = 'rsync -ravzp {src} {dst}'.format(**{
+            'src': '/'.join([project_dir, 'files/maps']),
+            'dst': '/'.join([project_dir, 'instance/LinuxServer/UnrealTournament/Content/Paks/'])
+        })
+        self._invoke_command(cmd)
+
+        self.ctx.log['ut4'].info('Installing mutators...')
+        cmd = 'rsync -ravzp {src} {dst}'.format(**{
+            'src': '/'.join([project_dir, 'files/mutators']),
+            'dst': '/'.join([project_dir, 'instance/LinuxServer/UnrealTournament/Content/Paks/'])
+        })
+        self._invoke_command(cmd)
+
+    def _install_redirect_lines(self):
+        return self.install_redirect_lines()
+
+    def install_redirect_lines(self):
+        self.ctx.log['ut4'].info('Generating redirect references...')
+        redirect_protocol = self.ctx.config['app']['redirect_protocol']
+        redirect_url = self.ctx.config['app']['redirect_url']
+        project_dir = self.ctx.config['app']['project_dir']
+        mod_dir = '/'.join([project_dir, 'files'])
+        files = glob.iglob('{}/**/*.pak'.format(mod_dir))
+        redirect_lines = []
+        for idx, filename in enumerate(files):
+            if idx > 5:
+                break
+            md5sum = self._md5sum_file(filename)
+            p = pathlib.Path(filename)
+            relative_path = p.relative_to(mod_dir)
+            basename = p.name
+
+            line = '\
+RedirectReferences=(PackageName="{basename}",\
+PackageURLProtocol="{redirect_protocol}",\
+PackageURL="{redirect_url}/{relative_path}",\
+PackageChecksum="{md5sum}")'\
+.format(**{
+    'basename': basename,
+    'redirect_protocol': redirect_protocol,
+    'redirect_url': redirect_url,
+    'relative_path': relative_path,
+    'md5sum': md5sum
+})
+            self.ctx.log['ut4'].debug("redirect line = '%s'", line)
+            redirect_lines.append(line)
+            # end loop - for filename in files:
+
+        # START HERE --trying to dynamically add redirect references and
+        # gonna need to be able to work with the ini file, including support
+        # for duplicate keys. Alternatively, maybe do the redirect lines through
+        # a patch
+
+        # I found out the hard way, configobj doesn't seem the best suited for
+        # duplicate keys...
+
+        # install into Game.ini
+        game_ini_filepath = '/'.join([project_dir, 'files/config/Game.ini'])
+        # game_ini_data = configobj.ConfigObj(game_ini_filepath)
+        config = configparser.ConfigParser(
+            # game_ini_filepath,
+            # dict_type=MultiOrderedDict,
+            strict=False
+        )
+        s = config.sections()
+        print(s)
+        # config.set('/Script/UnrealTournament.UTBaseGameMode', 'RedirectReferences', line)
+        config.read(game_ini_filepath)
+        l = config.write(sys.stdout)
+        print('l', l)
+
+        
+    def _install_rulesets(self):
+        self.ctx.log['ut4'].info('Concatenating rulesets for game modes...')
+        project_dir = self.ctx.config['app']['project_dir']
+
+        src_dir = '/'.join([project_dir, 'files/rulesets'])
+        out_dir = '/'.join([project_dir, '/instance/LinuxServer/UnrealTournament/Saved/Rulesets'])
+        out_filename='/'.join([out_dir, 'ruleset.json'])
+
+        cmd = 'mkdir -pv {out_dir}'.format(**{
+            'out_dir': out_dir
+        })
+        self._invoke_command(cmd)
+
+        self.ctx.log['ut4'].info('out filename=%s', out_filename)
+        
+        # echo {\"rules\":[ > "$OUT_FILENAME"
+        cmd = 'echo {\"rules\":[ > "{out_filename}"'
+        self._invoke_command(cmd)
+
+        cmd = 'for f in "{src_dir}"/*.json ; do ; cat "$f" >> "{out_filename}" ; done'.format(
+            **{
+            'src_dir': src_dir,
+            'out_filename': out_filename
+        })
+        self._invoke_command(cmd)
+       
+        cmd = 'echo "]}}" >> "{out_filename}"'.format(
+            **{
+            'out_filename': out_filename
+        })
+        self._invoke_command(cmd)
+        self.ctx.log['ut4'].info('output ruleset is at "%s"', out_filename)
+        
+    def _invoke_command(self, cmd, msg=None):
+        assert isinstance(cmd, str), 'cmd input must be string: %s'.format(cmd)
+        if msg is None:
+            msg = cmd
+        print(msg)
+        self.ctx.log['ut4'].info('running cmd: %s', cmd)
+        cwd = None # todo(mg) ?
+        # os.system(cmd)
+
+    def _md5sum_file(self, filename):
+        with open(filename, mode='rb') as f:
+            d = hashlib.md5()
+            for buf in iter(partial(f.read, 128), b''):
+                d.update(buf)
+            h = d.hexdigest()
+            return h
+
+    def _validate_env_vars(self):
+        variable_names = (
+            'project_dir',
+            'download_url',
+            'download_filename',
+            'download_md5',
+            'redirect_protocol',
+            'redirect_url',
+            'remote_game_host',
+            'remote_game_directory',
+            'remote_redirect_host'
+        )
+        for name in variable_names:
+            value = self.ctx.config['app'][name]
+            self.ctx.log['ut4'].info('%s: %s', name, value) 
+
+        i = input('Continue with above configuration? (y/N):')
+        if i.lower() != 'y':
+            self.ctx.log['ut4'].info('Doing nothing.')
+            return False
+        self.ctx.log['ut4'].info('Continuing.')
+        return True
+
+class MultiOrderedDict(collections.OrderedDict):
+    def __setitem__(self, key, value):
+        if isinstance(value, list) and key in self:
+            self[key].extend(value)
+        else:
+            super().__setitem__(key, value)
+