zavage-software/smileyface
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: refactor black and isort

Browse Source
This commit is contained in:
Mathew Guest
2024-08-02 05:51:33 -06:00
parent c69fc19b1d
commit 49f4816c8d
17 changed files with 768 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files
smileyface/__init__.py
+13 -9
View File
@@ -2,18 +2,20 @@ import logging
import sys
# Module parameters and constants
APP_NAME = 'SmileyFace Unreal Tournament 4 Server Panel'
APP_AUTHOR = 'Mathew Guest'
APP_VERSION = '0.1.0'
APP_NAME = "SmileyFace Unreal Tournament 4 Server Panel"
APP_AUTHOR = "Mathew Guest"
APP_VERSION = "0.1.0"
APP_CONFIG_FILENAME = 'config.ini'
APP_CONFIG_FILENAME = "config.ini"
# config.spec is relative to the module src directory and is the
# config specification (structure, names, and types of config file)
APP_CONFIGSPEC_FILENAME = 'config.spec'
APP_CONFIGSPEC_FILENAME = "config.spec"
# Check and gracefully fail if the user needs to install a 3rd-party dep.
required_lib_names = ['appdirs', 'configobj', 'colorlog']
required_lib_names = ["appdirs", "configobj", "colorlog"]
def check_env_has_dependencies(required_lib_names):
"""
Attempts to import each module and gracefully fails if it doesn't
@@ -24,15 +26,17 @@ def check_env_has_dependencies(required_lib_names):
try:
__import__(libname)
except ImportError as ex:
print('missing third-part library: ', ex, file=sys.stderr)
print("missing third-part library: ", ex, file=sys.stderr)
rc = False
except Exception as ex:
print(ex, type(ex))
rc = False
return rc
if not check_env_has_dependencies(required_lib_names):
print('refusing to load program without installed dependencies', file=sys.stderr)
raise ImportError('python environment needs third-party dependencies installed')
print("refusing to load program without installed dependencies", file=sys.stderr)
raise ImportError("python environment needs third-party dependencies installed")
# Exposed from sub-modules:
from .app import start_app
smileyface/_util.py
+4 -4
View File
@@ -1,11 +1,11 @@
import hashlib
import functools
import hashlib
def md5sum_file(filename):
with open(filename, mode='rb') as f:
with open(filename, mode="rb") as f:
d = hashlib.md5()
for buf in iter(functools.partial(f.read, 128), b''):
for buf in iter(functools.partial(f.read, 128), b""):
d.update(buf)
h = d.hexdigest()
return h
smileyface/app.py
+23 -38
View File
@@ -1,63 +1,49 @@
from . import hub_machine
from . import datalayer
from . import scrape_latest
import app_skellington
from app_skellington import _util
from . import (
datalayer,
hub_machine,
scrape_latest,
)
class SmileyFace(app_skellington.ApplicationContainer):
def __init__(self, *args, **kwargs):
filename = 'config.spec'
filename = "config.spec"
self.configspec_filepath = _util.get_asset(__name__, filename)
config_filepath = self._get_config_filepath(
'smileyface-ut4',
'',
'hub-config.ini'
)
config_filepath = self._get_config_filepath("smileyface-ut4", "", "hub-config.ini")
super().__init__(
configspec_filepath=self.configspec_filepath,
configini_filepath=config_filepath,
app_name = 'SmileyFace UT4 Server Panel',
app_author = 'Mathew Guest',
app_version = '0.1',
app_name="SmileyFace UT4 Server Panel",
app_author="Mathew Guest",
app_version="0.1",
*args,
**kwargs
**kwargs,
)
def _cli_options(self):
pass
def _command_menu(self):
sm_root = self.cli.init_submenu('command')
_util.register_class_as_commands(
self, sm_root,
hub_machine.UT4ServerMachine
)
sm_root = self.cli.init_submenu("command")
_util.register_class_as_commands(self, sm_root, hub_machine.UT4ServerMachine)
sm_scrape = sm_root.create_submenu('scrape')
_util.register_class_as_commands(
self, sm_scrape,
scrape_latest.ScrapeUt4Pugs
)
sm_scrape = sm_root.create_submenu("scrape")
_util.register_class_as_commands(self, sm_scrape, scrape_latest.ScrapeUt4Pugs)
_util.register_class_as_commands(
self, sm_scrape,
scrape_latest.ScrapeUtcc
)
_util.register_class_as_commands(self, sm_scrape, scrape_latest.ScrapeUtcc)
_util.register_class_as_commands(
self, sm_scrape,
scrape_latest.LocalFs
)
_util.register_class_as_commands(self, sm_scrape, scrape_latest.LocalFs)
def _services(self):
self['model'] = lambda: hub_machine.UTServerMachine(self.ctx)
self["model"] = lambda: hub_machine.UTServerMachine(self.ctx)
self.dal = datalayer.DataLayer(self.ctx)
self['dal'] = lambda: self.dal
self['datalayer'] = lambda: datalayer.DbFuncs(self.ctx, self.dal)
self["dal"] = lambda: self.dal
self["datalayer"] = lambda: datalayer.DbFuncs(self.ctx, self.dal)
# self['localfs'] = lambda: datalayer.LocalFs(self.ctx, datalayer)
@@ -67,7 +53,7 @@ class SmileyFace(app_skellington.ApplicationContainer):
def invoke_from_cli(self):
rc = self.load_command()
if not rc:
print('Invalid command. Try -h for usage')
print("Invalid command. Try -h for usage")
return
# load config
self.invoke_command()
@@ -136,10 +122,9 @@ Typical Usage:
./ut4-server-ctl.sh upload-server
./ut4-server-ctl.sh restart-server
"""
print(s)
print(s)
def start_app():
app = SmileyFace()
app.invoke_from_cli()
smileyface/datalayer/datalayer.py
+8 -11
View File
@@ -1,8 +1,9 @@
from smileyface import myutil
import os
import sqlite3
import appdirs
import sqlite3
import os
from smileyface import myutil
class DataLayer:
@@ -17,10 +18,10 @@ class DataLayer:
return self._db_conn
def _create_db_connection(self):
local_db_filename = self.ctx.config['app']['sqlite_filename']
appdir = appdirs.user_data_dir('smileyface')
local_db_filename = self.ctx.config["app"]["sqlite_filename"]
appdir = appdirs.user_data_dir("smileyface")
fullpath = os.path.join(appdir, local_db_filename)
self.ctx.log['ut4'].info('sqlite3 filename: %s', fullpath)
self.ctx.log["ut4"].info("sqlite3 filename: %s", fullpath)
myutil.ensure_dir_exists(fullpath)
@@ -28,9 +29,5 @@ class DataLayer:
return db
def commit(self):
self.ctx.log['db'].info('commit()')
self.ctx.log["db"].info("commit()")
self.db_conn.commit()
smileyface/datalayer/db_ops.py
+18 -23
View File
@@ -1,12 +1,12 @@
from smileyface import myutil
from smileyface import structs
import datetime
import os
import app_skellington._util as apputil
import appdirs
import datetime
import os
import sqlparse
from smileyface import myutil, structs
class DbFuncs:
def __init__(self, ctx, dal):
@@ -14,7 +14,7 @@ class DbFuncs:
self.dal = dal
def create_tables(self):
sql_filename = apputil.get_asset(__name__, 'create_schema.sql')
sql_filename = apputil.get_asset(__name__, "create_schema.sql")
with open(sql_filename) as fp:
contents_sql = fp.read()
stmts = sqlparse.split(contents_sql)
@@ -22,7 +22,7 @@ class DbFuncs:
conn = self.dal.db_conn
curs = conn.cursor()
for stmt in stmts:
print('----')
print("----")
print(stmt)
curs.execute(stmt)
@@ -31,9 +31,9 @@ class DbFuncs:
def truncate_tables(self):
conn = self.dal.db_conn
curs = conn.cursor()
sql = '''
sql = """
truncate file_paks;
'''
"""
for stmt in sqlparse.split(sql):
curs.execute(stmt)
self.dal.commit()
@@ -43,7 +43,7 @@ truncate file_paks;
# NOTE(MG) datetime parameter logic could be improved and more complete
conn = self.dal.db_conn
curs = conn.cursor()
sql = '''
sql = """
insert into file_paks (
file_pak_id,
fullpath,
@@ -63,12 +63,12 @@ on conflict(filename) do update set
md5sum = excluded.md5sum,
--created at does not update
record_updated_at = datetime('now', 'localtime')
'''
"""
args = (
record.file_pak_id,
record.fullpath,
record.filename,
record.md5sum
record.md5sum,
# record.record_created_at,
# record.record_updated_at
)
@@ -80,7 +80,7 @@ on conflict(filename) do update set
# validation data src
conn = self.dal.db_conn
curs = conn.cursor()
sql = '''
sql = """
update file_paks
set
validated_state = ?,
@@ -90,7 +90,7 @@ set
where
file_pak_id = ?
'''
"""
args = (validate_state, src, remote_src_md5, rec_id)
curs.execute(sql, args)
conn.commit()
@@ -98,14 +98,14 @@ where
def query_filepak(self, filename):
conn = self.dal.db_conn
curs = conn.cursor()
sql = '''
sql = """
select
file_pak_id, fullpath, filename,
md5sum, record_created_at, record_updated_at
from
file_paks
where
lower(filename) = lower(?)'''
lower(filename) = lower(?)"""
args = (filename,)
# print(sql)
curs.execute(sql, args)
@@ -126,15 +126,14 @@ where
elif len(output) == 1:
return output[0]
elif len(output) > 1:
input('<breakpoint> unexpected two rows returned from db when expecting to be unique')
input("<breakpoint> unexpected two rows returned from db when expecting to be unique")
return output
return output
def query_invalid_filepaks(self):
conn = self.dal.db_conn
curs = conn.cursor()
sql = '''
sql = """
select
file_pak_id, fullpath, filename,
md5sum,
@@ -143,7 +142,7 @@ select
from
file_paks
where
lower(filename) = lower(?)'''
lower(filename) = lower(?)"""
args = (filename,)
# print(sql)
curs.execute(sql, args)
@@ -160,7 +159,3 @@ where
filepak.record_updated_at = r[8]
output.append(filepak)
return output
smileyface/gameconfig_edit.py
+21 -27
View File
@@ -1,6 +1,7 @@
import configparser
import re
class UnrealIniFile:
def __init__(self, filename=None):
self._config = None
@@ -14,12 +15,11 @@ class UnrealIniFile:
def filename(self, val):
self._filename = val
if val is not None:
self._config = configparser.RawConfigParser(
strict=False
)
self._config.optionxform = str # Trick to preserve case in key names
self._config = configparser.RawConfigParser(strict=False)
self._config.optionxform = str # Trick to preserve case in key names
self._config.read(self._filename)
class GameIniSpecial:
def __init__(self, filename):
self._redirect_lines = []
@@ -37,24 +37,23 @@ class GameIniSpecial:
def clear_redirect_references(self):
self._redirect_lines = []
def add_redirect_reference(
self, pkg_basename, redirect_url, redirect_protocol,
relative_path, md5sum
):
def add_redirect_reference(self, pkg_basename, redirect_url, redirect_protocol, relative_path, md5sum):
args = {
'pkg_basename': pkg_basename,
'redirect_protocol': redirect_protocol,
'redirect_url': redirect_url,
'relative_path': relative_path,
'md5sum': md5sum
"pkg_basename": pkg_basename,
"redirect_protocol": redirect_protocol,
"redirect_url": redirect_url,
"relative_path": relative_path,
"md5sum": md5sum,
}
########### START multi-line awkward indent
########### START multi-line awkward indent
line = '\
RedirectReferences=(PackageName="{pkg_basename}",\
PackageURLProtocol="{redirect_protocol}",\
PackageURL="{redirect_url}/{relative_path}",\
PackageChecksum="{md5sum}")'.format(**args)
########### END multi-line awkward indent
PackageChecksum="{md5sum}")'.format(
**args
)
########### END multi-line awkward indent
return self.add_redirect_reference_line(line)
@@ -64,19 +63,14 @@ PackageChecksum="{md5sum}")'.format(**args)
def write(self, fp):
newcontents = None
with open(self.filename, 'r') as inifile:
with open(self.filename, "r") as inifile:
curcontents = inifile.read()
lines_str = '\n'.join(self._redirect_lines)
newcontents = re.sub(
'RedirectReferences = :PARAM:',
lines_str,
curcontents
)
lines_str = "\n".join(self._redirect_lines)
newcontents = re.sub("RedirectReferences = :PARAM:", lines_str, curcontents)
has_data = True
if has_data:
with open(self.filename, 'w') as fp:
with open(self.filename, "w") as fp:
fp.write(newcontents)
smileyface/hub_machine.py
+197 -238
View File
@@ -1,10 +1,4 @@
from .gameconfig_edit import UnrealIniFile, GameIniSpecial
from ._util import md5sum_file
from . import myutil
from . import structs
import collections
import configobj
import configparser
import datetime
import glob
@@ -14,6 +8,12 @@ import subprocess
import sys
import time
import configobj
from . import myutil, structs
from ._util import md5sum_file
from .gameconfig_edit import GameIniSpecial, UnrealIniFile
class UT4ServerMachine:
def __init__(self, ctx, datalayer):
@@ -23,8 +23,6 @@ class UT4ServerMachine:
if not self._validate_env_vars():
sys.exit(1)
def oneclickdeploy(self):
self.generate_instance()
self.upload_redirects()
@@ -34,7 +32,7 @@ class UT4ServerMachine:
"""
Deletes the generated instance on the local machine.
"""
self.ctx.log['ut4'].info('Clearing .pak folder...')
self.ctx.log["ut4"].info("Clearing .pak folder...")
cmd = 'rm -rv "$PROJECT_DIR"/instance/LinuxServer/UnrealTournament/Content/Paks/*'
self._invoke_command(cmd)
@@ -42,61 +40,46 @@ class UT4ServerMachine:
"""
Create required directories which the user installs maps, mutators, and config to.
"""
dirs = (
'base',
'files/config',
'files/maps',
'files/mutators',
'files/rulesets',
'files/unused'
)
project_dir = self.ctx.config['app']['project_dir']
dirs = ("base", "files/config", "files/maps", "files/mutators", "files/rulesets", "files/unused")
project_dir = self.ctx.config["app"]["project_dir"]
if len(project_dir.strip()) == 0:
project_dir = '.'
print('project_dir:', project_dir)
fullpaths = [
'/'.join([project_dir, d]) for d in dirs
]
project_dir = "."
print("project_dir:", project_dir)
fullpaths = ["/".join([project_dir, d]) for d in dirs]
for fp in fullpaths:
cmd = 'mkdir -p {}'.format(fp)
cmd = "mkdir -p {}".format(fp)
self._invoke_command(cmd)
def download_linux_server(self, x):
"""
Download the latest Linux Unreal Tournament 4 Server from Epic
"""
self.ctx.log['ut4'].info('Downloading Linux Server Binary from Epic.')
self.ctx.log["ut4"].info("Downloading Linux Server Binary from Epic.")
def download_logs(self):
"""
Download the logs from the target hub.
"""
config_dir = self.ctx.config['app']['config_dir']
remote_game_host = self.ctx.config['app']['remote_game_host']
remote_game_dir = self.ctx.config['app']['remote_game_dir']
config_dir = self.ctx.config["app"]["config_dir"]
remote_game_host = self.ctx.config["app"]["remote_game_host"]
remote_game_dir = self.ctx.config["app"]["remote_game_dir"]
self.ctx.log['ut4'].info('Downloading instance logs from target hub.')
cmd = '''
self.ctx.log["ut4"].info("Downloading instance logs from target hub.")
cmd = """
rsync -ravzp {remote_game_host}:{remote_game_dir}/LinuxServer/UnrealTournament/Saved/Logs/ {config_dir}/downloaded-logs/
'''\
.format(**{
'config_dir': config_dir,
'remote_game_host': remote_game_host,
'remote_game_dir': remote_game_dir
})
""".format(
**{"config_dir": config_dir, "remote_game_host": remote_game_host, "remote_game_dir": remote_game_dir}
)
self._invoke_command(cmd)
# Delete logs on remote game server if successfully transferred to local:
self.ctx.log['ut4'].info('')
cmd = '''
self.ctx.log["ut4"].info("")
cmd = """
ssh {remote_game_host} rm {remote_game_dir}/LinuxServer/UnrealTournament/Saved/Logs/* -r'
'''\
.format(**{
'config_dir': config_dir,
'remote_game_host': remote_game_host,
'remote_game_dir': remote_game_dir
})
""".format(
**{"config_dir": config_dir, "remote_game_host": remote_game_host, "remote_game_dir": remote_game_dir}
)
# self._invoke_command(cmd)
def generate_instance(self):
@@ -104,34 +87,25 @@ ssh {remote_game_host} rm {remote_game_dir}/LinuxServer/UnrealTournament/Saved/L
Takes the current coniguration and outputs the application files which
can be copied to the server.
"""
self.ctx.log['ut4'].info('Generating server instance from custom files...')
project_dir = self.ctx.config['app']['project_dir']
self.ctx.log["ut4"].info("Generating server instance from custom files...")
project_dir = self.ctx.config["app"]["project_dir"]
# rsync
src = '/'.join([project_dir, 'base/LinuxServer'])
dst = '/'.join([project_dir, 'instance/'])
cmd = 'rsync -ravzp {src} {dst}'.format(**{
'src': src,
'dst': dst
})
src = "/".join([project_dir, "base/LinuxServer"])
dst = "/".join([project_dir, "instance/"])
cmd = "rsync -ravzp {src} {dst}".format(**{"src": src, "dst": dst})
self._invoke_command(cmd)
# cp 1
src = '/'.join([project_dir, 'start-server.sh'])
dst = '/'.join([project_dir, 'instance/'])
cmd = 'cp {src} {dst}'.format(**{
'src': src,
'dst': dst
})
src = "/".join([project_dir, "start-server.sh"])
dst = "/".join([project_dir, "instance/"])
cmd = "cp {src} {dst}".format(**{"src": src, "dst": dst})
self._invoke_command(cmd)
# cp 2
src = '/'.join([project_dir, 'stop-server.sh'])
dst = '/'.join([project_dir, 'instance/'])
cmd = 'cp {src} {dst}'.format(**{
'src': src,
'dst': dst
})
src = "/".join([project_dir, "stop-server.sh"])
dst = "/".join([project_dir, "instance/"])
cmd = "cp {src} {dst}".format(**{"src": src, "dst": dst})
self._invoke_command(cmd)
if self._needs_first_run():
@@ -149,35 +123,35 @@ ssh {remote_game_host} rm {remote_game_dir}/LinuxServer/UnrealTournament/Saved/L
"""
Flip on the target hub on for Fragging!
"""
self.ctx.log['ut4'].info('Starting hub...')
self.ctx.log["ut4"].info("Starting hub...")
cmd = '''
cmd = """
ssh {remote_game_host} {remote_game_dir}/start-server.sh
'''
"""
self._invoke_command(cmd)
def stop_server(self):
"""
Stop UT4 Hub processes on the server.
"""
self.ctx.log['ut4'].info('Stopping hub.')
cmd = '''
self.ctx.log["ut4"].info("Stopping hub.")
cmd = """
ssh {remote_game_host} {remote_game_dir}/stop-server.sh
'''
"""
self._invoke_command(cmd)
def upload_redirects(self):
"""
Upload paks to redirect server.
"""
self.ctx.log['ut4'].info('Uploading redirects (maps, mutators, etc.) to target hub.')
self.ctx.log["ut4"].info("Uploading redirects (maps, mutators, etc.) to target hub.")
project_dir = self.ctx.config['app']['project_dir']
project_dir = self.ctx.config["app"]["project_dir"]
# paks_dir = os.path.join(project_dir, 'instance/LinuxServer/UnrealTournament/Content/Paks/')
paks_dir = os.path.join(project_dir, 'files/') # trailing slash required
remote_redirect_host = self.ctx.config['app']['remote_redirect_host']
paks_dir = os.path.join(project_dir, "files/") # trailing slash required
remote_redirect_host = self.ctx.config["app"]["remote_redirect_host"]
cwd = project_dir
cmd = '''
cmd = """
rsync -rivz \
--delete \
--exclude "*.md5" \
@@ -186,11 +160,12 @@ rsync -rivz \
--exclude Mods.db \
{paks_dir} {remote_redirect_host}
'''\
.format(**{
'paks_dir': paks_dir,
'remote_redirect_host': remote_redirect_host,
})
""".format(
**{
"paks_dir": paks_dir,
"remote_redirect_host": remote_redirect_host,
}
)
# subprocess.run(cmd, cwd=cwd) # should be invoke_command? no because gui will need subprocess.run
self._invoke_command(cmd)
@@ -199,61 +174,66 @@ rsync -rivz \
self._redirect_chown()
def _redirect_hide_passwords(self):
project_dir = self.ctx.config['app']['project_dir']
remote_redirect_host = self.ctx.config['app']['remote_redirect_host']
project_dir = self.ctx.config["app"]["project_dir"]
remote_redirect_host = self.ctx.config["app"]["remote_redirect_host"]
# (on the server):
gameini="/srv/ut4-redirect.zavage.net/config/Game.ini"
engineini="/srv/ut4-redirect.zavage.net/config/Engine.ini"
gameini = "/srv/ut4-redirect.zavage.net/config/Game.ini"
engineini = "/srv/ut4-redirect.zavage.net/config/Engine.ini"
cmd = '''
cmd = """
ssh mathewguest.com \
sed -i /ServerInstanceID=/c\ServerInstanceID=Hidden {gameini}
'''.format(gameini=gameini)
""".format(
gameini=gameini
)
self._invoke_command(cmd)
cmd = '''
cmd = """
ssh mathewguest.com \
sed -i /RconPassword=/c\RconPassword=Hidden {engineini}
'''.format(engineini=engineini)
""".format(
engineini=engineini
)
self._invoke_command(cmd)
def _redirect_upload_script(self):
project_dir = self.ctx.config['app']['project_dir']
remote_redirect_host = self.ctx.config['app']['remote_redirect_host']
cmd = '''
project_dir = self.ctx.config["app"]["project_dir"]
remote_redirect_host = self.ctx.config["app"]["remote_redirect_host"]
cmd = """
rsync -vz \
{project_dir}/ut4-server-ctl.sh \
{remote_redirect_host}
'''\
.format(**{
'project_dir': project_dir,
'remote_redirect_host': remote_redirect_host,
})
""".format(
**{
"project_dir": project_dir,
"remote_redirect_host": remote_redirect_host,
}
)
# subprocess.run(cmd, cwd=cwd) # should be invoke_command? no because gui will need subprocess.run
self._invoke_command(cmd)
def _redirect_chown(self):
project_dir = self.ctx.config['app']['project_dir']
remote_redirect_host = self.ctx.config['app']['remote_redirect_host']
project_dir = self.ctx.config["app"]["project_dir"]
remote_redirect_host = self.ctx.config["app"]["remote_redirect_host"]
cmd = '''
cmd = """
ssh mathewguest.com \
chown http:http /srv/ut4-redirect.zavage.net -R
'''
"""
self._invoke_command(cmd)
def upload_server(self):
"""
Upload all required game files to the hub server.
"""
self.ctx.log['ut4'].info('Uploading customized server')
project_dir = self.ctx.config['app']['project_dir']
remote_game_host = self.ctx.config['app']['remote_game_host']
remote_game_dir = self.ctx.config['app']['remote_game_dir']
self.ctx.log["ut4"].info("Uploading customized server")
project_dir = self.ctx.config["app"]["project_dir"]
remote_game_host = self.ctx.config["app"]["remote_game_host"]
remote_game_dir = self.ctx.config["app"]["remote_game_dir"]
cwd = None
# transfer #1
cmd = '''
cmd = """
rsync -raivzp \
--delete \
--exclude ".KEEP" \
@@ -266,88 +246,72 @@ rsync -raivzp \
--exclude "Saved/Logs/*" \
{project_dir}/instance/ \
{remote_game_host}:{remote_game_dir}
'''\
.format(**{
'project_dir': project_dir,
'remote_game_host': remote_game_host,
'remote_game_dir': remote_game_dir
})
cmd = cmd.replace(' ', '')
""".format(
**{"project_dir": project_dir, "remote_game_host": remote_game_host, "remote_game_dir": remote_game_dir}
)
cmd = cmd.replace(" ", "")
# subprocess.run(cmd, cwd=cwd) # should be invoke_command? no because gui will need subprocess.run
self._invoke_command(cmd)
# transfer #2
cmd = '''
cmd = """
rsync -avzp \
{project_dir}/ut4-server-ctl.sh \
{remote_game_host}:{remote_game_dir}
'''\
.format(**{
'project_dir': project_dir,
'remote_game_host': remote_game_host,
'remote_game_dir': remote_game_dir
})
""".format(
**{"project_dir": project_dir, "remote_game_host": remote_game_host, "remote_game_dir": remote_game_dir}
)
# subprocess.run(cmd, cwd=cwd)
self._invoke_command(cmd)
# transfer #3
cmd = '''
cmd = """
scp \
{project_dir}/instance/ut4-server.service \
{remote_game_host}:/etc/systemd/system/
'''\
.format(**{
'project_dir': project_dir,
'remote_game_host': remote_game_host,
'remote_game_dir': remote_game_dir
})
""".format(
**{"project_dir": project_dir, "remote_game_host": remote_game_host, "remote_game_dir": remote_game_dir}
)
# subprocess.run(cmd, cwd=cwd)
self._invoke_command(cmd)
# transfer #4
cmd = '''
cmd = """
ssh {remote_game_host} \
chown ut4:ut4 {remote_game_dir} -R
'''.format(**{
'remote_game_host': remote_game_host,
'remote_game_dir': remote_game_dir
})
""".format(
**{"remote_game_host": remote_game_host, "remote_game_dir": remote_game_dir}
)
# subprocess.run(cmd, cwd=cwd)
self._invoke_command(cmd)
# Fix +x permissions on bash scripts
cmd = '''
cmd = """
ssh {remote_game_host} \
chmod +x \
{remote_game_dir}/start-server.sh \
{remote_game_dir}/stop-server.sh
'''.format(**{
'remote_game_host': remote_game_host,
'remote_game_dir': remote_game_dir
})
""".format(
**{"remote_game_host": remote_game_host, "remote_game_dir": remote_game_dir}
)
# subprocess.run(cmd, cwd=cwd)
self._invoke_command(cmd)
def _first_run(self):
self.ctx.log['ut4'].info('Starting instance once to get UID.')
self.ctx.log['ut4'].info('Unfortunately, this takes 20 seconds. Just wait.')
self.ctx.log["ut4"].info("Starting instance once to get UID.")
self.ctx.log["ut4"].info("Unfortunately, this takes 20 seconds. Just wait.")
# Make binary executable:
bin_name = 'UE4Server-Linux-Shipping'
project_dir = self.ctx.config['app']['project_dir']
# Make binary executable:
bin_name = "UE4Server-Linux-Shipping"
project_dir = self.ctx.config["app"]["project_dir"]
cwd = '{project_dir}/instance/LinuxServer/Engine/Binaries/Linux'\
.format(project_dir=project_dir)
target_file = '{cwd}/{bin_name}'.format(cwd=cwd, bin_name=bin_name)
cmd = ['chmod', '770', target_file]
cwd = "{project_dir}/instance/LinuxServer/Engine/Binaries/Linux".format(project_dir=project_dir)
target_file = "{cwd}/{bin_name}".format(cwd=cwd, bin_name=bin_name)
cmd = ["chmod", "770", target_file]
p = subprocess.run(cmd)
cmd = ['./'+bin_name, 'UnrealTournament', 'UT-Entry?Game=Lobby', '-log']
cmd = ["./" + bin_name, "UnrealTournament", "UT-Entry?Game=Lobby", "-log"]
p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE)
try:
@@ -356,68 +320,68 @@ ssh {remote_game_host} \
p.kill()
stdout, stderr = p.communicate()
self.ctx.log['ut4'].info('sleeping 20 seconds and then we\'ll kill the server we started just now.')
self.ctx.log["ut4"].info("sleeping 20 seconds and then we'll kill the server we started just now.")
# # TODO(MG) get uid and export
def _install_config(self):
files = (
'Game.ini',
'Engine.ini'
)
project_dir = self.ctx.config['app']['project_dir']
config_dir = self.ctx.config['app']['config_dir']
files = ("Game.ini", "Engine.ini")
project_dir = self.ctx.config["app"]["project_dir"]
config_dir = self.ctx.config["app"]["config_dir"]
for fn in files:
self.ctx.log['ut4'].info('Installing file: %s', fn)
self.ctx.log["ut4"].info("Installing file: %s", fn)
src = os.path.join(config_dir, fn)
dst = os.path.join(project_dir, 'instance/LinuxServer/UnrealTournament/Saved/Config/LinuxServer', fn)
cmd = 'cp {src} {dst}'.format(**{
'src': src,
'dst': dst
})
dst = os.path.join(project_dir, "instance/LinuxServer/UnrealTournament/Saved/Config/LinuxServer", fn)
cmd = "cp {src} {dst}".format(**{"src": src, "dst": dst})
self._invoke_command(cmd)
# Monkey-patch Game.ini to ensure it has a place for RedirectReferences
if fn == 'Game.ini':
if fn == "Game.ini":
ini = UnrealIniFile(dst)
sect_name = '/Script/UnrealTournament.UTBaseGameMode'
opt_name = 'RedirectReferences'
sect_name = "/Script/UnrealTournament.UTBaseGameMode"
opt_name = "RedirectReferences"
if not ini._config.has_section(sect_name):
ini._config.add_section(sect_name)
if not ini._config.has_option(sect_name, opt_name):
ini._config.set(sect_name, opt_name, ':PARAM:')
with open(dst, 'w') as fp:
ini._config.set(sect_name, opt_name, ":PARAM:")
with open(dst, "w") as fp:
ini._config.write(fp)
def _install_paks(self):
project_dir = self.ctx.config['app']['project_dir']
project_dir = self.ctx.config["app"]["project_dir"]
self.ctx.log['ut4'].info('Installing maps...')
cmd = 'rsync -ravzp {src} {dst}'.format(**{
'src': '/'.join([project_dir, 'files/maps/']),
'dst': '/'.join([project_dir, 'instance/LinuxServer/UnrealTournament/Content/Paks/'])
})
self.ctx.log["ut4"].info("Installing maps...")
cmd = "rsync -ravzp {src} {dst}".format(
**{
"src": "/".join([project_dir, "files/maps/"]),
"dst": "/".join([project_dir, "instance/LinuxServer/UnrealTournament/Content/Paks/"]),
}
)
self._invoke_command(cmd)
self.ctx.log['ut4'].info('Installing mutators...')
cmd = 'rsync -ravzp {src} {dst}'.format(**{
'src': '/'.join([project_dir, 'files/mutators/']),
'dst': '/'.join([project_dir, 'instance/LinuxServer/UnrealTournament/Content/Paks/'])
})
self.ctx.log["ut4"].info("Installing mutators...")
cmd = "rsync -ravzp {src} {dst}".format(
**{
"src": "/".join([project_dir, "files/mutators/"]),
"dst": "/".join([project_dir, "instance/LinuxServer/UnrealTournament/Content/Paks/"]),
}
)
self._invoke_command(cmd)
def _install_redirect_lines(self):
self.ctx.log['ut4'].info('Generating redirect references...')
self.ctx.log["ut4"].info("Generating redirect references...")
redirect_protocol = self.ctx.config['app']['redirect_protocol']
redirect_url = self.ctx.config['app']['redirect_url']
project_dir = self.ctx.config['app']['project_dir']
mod_dir = '/'.join([project_dir, 'files'])
redirect_protocol = self.ctx.config["app"]["redirect_protocol"]
redirect_url = self.ctx.config["app"]["redirect_url"]
project_dir = self.ctx.config["app"]["project_dir"]
mod_dir = "/".join([project_dir, "files"])
game_ini_filepath = '/'.join([project_dir, 'instance/LinuxServer/UnrealTournament/Saved/Config/LinuxServer/Game.ini'])
game_ini_filepath = "/".join(
[project_dir, "instance/LinuxServer/UnrealTournament/Saved/Config/LinuxServer/Game.ini"]
)
game_ini = GameIniSpecial(game_ini_filepath)
game_ini.clear_redirect_references()
files = glob.glob('{}/**/*.pak'.format(mod_dir))
files = glob.glob("{}/**/*.pak".format(mod_dir))
redirect_lines = []
for idx, filename in enumerate(files):
# if idx > 5:
@@ -435,90 +399,85 @@ ssh {remote_game_host} \
print(ex)
continue
line = game_ini.add_redirect_reference(**{
'pkg_basename': pkg_basename,
'redirect_protocol': redirect_protocol,
'redirect_url': redirect_url,
'relative_path': relative_path,
'md5sum': md5sum
})
line = game_ini.add_redirect_reference(
**{
"pkg_basename": pkg_basename,
"redirect_protocol": redirect_protocol,
"redirect_url": redirect_url,
"relative_path": relative_path,
"md5sum": md5sum,
}
)
self.ctx.log['ut4'].debug("redirect line = '%s'", line)
self.ctx.log["ut4"].debug("redirect line = '%s'", line)
data = game_ini.write(sys.stdout)
def _install_rulesets(self):
self.ctx.log['ut4'].info('Concatenating rulesets for game modes...')
project_dir = self.ctx.config['app']['project_dir']
self.ctx.log["ut4"].info("Concatenating rulesets for game modes...")
project_dir = self.ctx.config["app"]["project_dir"]
src_dir = '/'.join([project_dir, 'files/rulesets'])
out_dir = '/'.join([project_dir, '/instance/LinuxServer/UnrealTournament/Saved/Config/Rulesets'])
out_filename='/'.join([out_dir, 'ruleset.json'])
src_dir = "/".join([project_dir, "files/rulesets"])
out_dir = "/".join([project_dir, "/instance/LinuxServer/UnrealTournament/Saved/Config/Rulesets"])
out_filename = "/".join([out_dir, "ruleset.json"])
cmd = 'mkdir -pv {out_dir}'.format(**{
'out_dir': out_dir
})
cmd = "mkdir -pv {out_dir}".format(**{"out_dir": out_dir})
self._invoke_command(cmd)
self.ctx.log['ut4'].info('out filename=%s', out_filename)
self.ctx.log["ut4"].info("out filename=%s", out_filename)
# echo {\"rules\":[ > "$OUT_FILENAME"
cmd = "echo '{{\"rules\":[' > \"{out_filename}\"".format(out_filename=out_filename)
cmd = 'echo \'{{"rules":[\' > "{out_filename}"'.format(out_filename=out_filename)
self._invoke_command(cmd)
cmd = 'for f in "{src_dir}"/*.json ; do cat "$f" >> "{out_filename}" ; done'.format(
**{
'src_dir': src_dir,
'out_filename': out_filename
})
**{"src_dir": src_dir, "out_filename": out_filename}
)
self._invoke_command(cmd)
cmd = 'echo "]}}" >> "{out_filename}"'.format(
**{
'out_filename': out_filename
})
cmd = 'echo "]}}" >> "{out_filename}"'.format(**{"out_filename": out_filename})
self._invoke_command(cmd)
self.ctx.log['ut4'].info('output ruleset is at "%s"', out_filename)
self.ctx.log["ut4"].info('output ruleset is at "%s"', out_filename)
def _invoke_command(self, cmd, msg=None):
assert isinstance(cmd, str), 'cmd input must be string: %s'.format(cmd)
assert isinstance(cmd, str), "cmd input must be string: %s".format(cmd)
if msg is None:
msg = cmd
print(msg)
self.ctx.log['ut4'].info('running cmd: %s', cmd)
cwd = None # todo(mg) ?
self.ctx.log["ut4"].info("running cmd: %s", cmd)
cwd = None # todo(mg) ?
os.system(cmd)
def _needs_first_run(self):
return False # TODO(MG): Hard-coded
return False # TODO(MG): Hard-coded
def _validate_env_vars(self):
variable_names = (
'project_dir',
'download_url',
'download_filename',
'download_md5',
'redirect_protocol',
'redirect_url',
'remote_game_host',
'remote_game_dir',
'remote_redirect_host'
"project_dir",
"download_url",
"download_filename",
"download_md5",
"redirect_protocol",
"redirect_url",
"remote_game_host",
"remote_game_dir",
"remote_redirect_host",
)
for name in variable_names:
value = self.ctx.config['app'][name]
self.ctx.log['ut4'].info('%s: %s', name, value)
value = self.ctx.config["app"][name]
self.ctx.log["ut4"].info("%s: %s", name, value)
i = input('Continue with above configuration? (y/N):')
if i.lower() != 'y':
self.ctx.log['ut4'].info('Doing nothing.')
i = input("Continue with above configuration? (y/N):")
if i.lower() != "y":
self.ctx.log["ut4"].info("Doing nothing.")
return False
self.ctx.log['ut4'].info('Continuing.')
self.ctx.log["ut4"].info("Continuing.")
return True
class MultiOrderedDict(collections.OrderedDict):
def __setitem__(self, key, value):
if isinstance(value, list) and key in self:
self[key].extend(value)
else:
super().__setitem__(key, value)
smileyface/myutil.py
+3 -3
View File
@@ -5,14 +5,14 @@ import os
def ensure_dir_exists(dirpath):
if dirpath is None:
return
if dirpath == '':
if dirpath == "":
return
dirpath = os.path.dirname(dirpath)
os.makedirs(dirpath, exist_ok=True)
def md5_file(filename):
with open(filename, 'rb') as fp:
with open(filename, "rb") as fp:
data = fp.read()
h = hashlib.md5(data).hexdigest()
return h
return h
smileyface/scrape_latest/__init__.py
+1 -1
View File
@@ -1,3 +1,3 @@
from .local_fs import *
from .scrape_utcc import *
from .scrape_ut4pugs import *
from .scrape_utcc import *
smileyface/scrape_latest/local_fs.py
+5 -6
View File
@@ -1,9 +1,8 @@
from smileyface import myutil
from smileyface import structs
import datetime
import os
from smileyface import myutil, structs
class LocalFs:
def __init__(self, ctx, datalayer):
@@ -14,12 +13,12 @@ class LocalFs:
self.datalayer.create_tables()
def load_md5s(self):
paks_dir = self.ctx.config['app']['project_dir']
maps_dir = os.path.join(paks_dir, 'files', 'maps')
paks_dir = self.ctx.config["app"]["project_dir"]
maps_dir = os.path.join(paks_dir, "files", "maps")
print(maps_dir)
self._load_md5_one_dir(maps_dir)
muts_dir = os.path.join(paks_dir, 'files', 'mutators')
muts_dir = os.path.join(paks_dir, "files", "mutators")
print(muts_dir)
self._load_md5_one_dir(muts_dir)
smileyface/scrape_latest/scrape_ut4pugs.py
+14 -14
View File
@@ -1,8 +1,8 @@
import selenium
import selenium.webdriver
URL_MUTATORS = 'https://ut4pugs.us/redirect-mutators'
URL_MAPS = 'https://ut4pugs.us/redirect-mutators'
URL_MUTATORS = "https://ut4pugs.us/redirect-mutators"
URL_MAPS = "https://ut4pugs.us/redirect-mutators"
class ScrapeUt4Pugs:
@@ -25,14 +25,14 @@ class ScrapeUt4Pugs:
self._check_pak_md5sums()
def _check_pak_md5sums(self):
tbl_of_mutators = self.browser.find_element_by_id('myTable')
tbl_of_mutators = self.browser.find_element_by_id("myTable")
print(tbl_of_mutators)
mut_rows = tbl_of_mutators.find_elements_by_xpath('tbody/tr')
mut_rows = tbl_of_mutators.find_elements_by_xpath("tbody/tr")
for r in mut_rows:
mut_cols = r.find_elements_by_xpath('td')
mut_cols = r.find_elements_by_xpath("td")
if len(mut_cols) != 3:
input('<breakpoint> at unexpected columns for mutator. received {}'.format(len(mut_cols)))
input("<breakpoint> at unexpected columns for mutator. received {}".format(len(mut_cols)))
mut_file = mut_cols[0]
mut_md5 = mut_cols[1]
mut_ini_line = mut_cols[2]
@@ -44,22 +44,22 @@ class ScrapeUt4Pugs:
local_file = self.datalayer.query_filepak(mut_file.text)
print(local_file)
if not local_file:
self.ctx.log['ut4'].warn('pak not found locally: %s', mut_file.text)
self.ctx.log["ut4"].warn("pak not found locally: %s", mut_file.text)
continue
local_md5 = local_file.md5sum
remote_md5 = mut_md5.text
if local_md5 != remote_md5:
input('<breakpoint> as mismatching md5!')
print('local: ', local_md5)
print('remote: ', remote_md5)
self.datalayer.mark_filepak_validated_state(local_file.file_pak_id, 'mismatch', 'ut4pugs', remote_md5)
input("<breakpoint> as mismatching md5!")
print("local: ", local_md5)
print("remote: ", remote_md5)
self.datalayer.mark_filepak_validated_state(local_file.file_pak_id, "mismatch", "ut4pugs", remote_md5)
else:
input('<breakpoint> as matching md5! good job')
self.datalayer.mark_filepak_validated_state(local_file.file_pak_id, 'valid', 'ut4pugs', None)
input("<breakpoint> as matching md5! good job")
self.datalayer.mark_filepak_validated_state(local_file.file_pak_id, "valid", "ut4pugs", None)
# print(r)
print('xxx')
print("xxx")
pass
pass
smileyface/scrape_latest/scrape_utcc.py
+1 -2
View File
@@ -1,7 +1,7 @@
import selenium
import selenium.webdriver
URL_CARDS_LIST = 'https://utcc.unrealpugs.com/content'
URL_CARDS_LIST = "https://utcc.unrealpugs.com/content"
class ScrapeUtcc:
@@ -21,4 +21,3 @@ class ScrapeUtcc:
def _scrape_list_of_content_cards(self):
self.browser.get(URL_CARDS_LIST)
smileyface/structs.py
+1 -1
View File
@@ -8,4 +8,4 @@ class FilePak:
self.filename = None
self.md5sum = None
self.record_created_at = None
self.record_updated_at = None
self.record_updated_at = None