latest version, incl. local sqlite db for managing files, check latest md5sum against ut4pugs, and some misc changes

smileyface/app.py

@@ -1,9 +1,11 @@
 from . import hub_machine
 from . import datalayer
+from . import scrape_latest
 
 import app_skellington
 from app_skellington import _util
 
+
 class SmileyFace(app_skellington.ApplicationContainer):
     def __init__(self, *args, **kwargs):
         filename = 'config.spec'
@@ -35,9 +37,29 @@ class SmileyFace(app_skellington.ApplicationContainer):
             hub_machine.UT4ServerMachine
         )
 
+        sm_scrape = sm_root.create_submenu('scrape')
+        _util.register_class_as_commands(
+            self, sm_scrape,
+            scrape_latest.ScrapeUt4Pugs
+        )
+
+        _util.register_class_as_commands(
+            self, sm_scrape,
+            scrape_latest.ScrapeUtcc
+        )
+
+        _util.register_class_as_commands(
+            self, sm_scrape,
+            scrape_latest.LocalFs
+        )
+
     def _services(self):
         self['model'] = lambda: hub_machine.UTServerMachine(self.ctx)
-        self['datalayer'] = lambda: datalayer.DataLayer(self.ctx)
+        self.dal = datalayer.DataLayer(self.ctx)
+        self['dal'] = lambda: self.dal
+        self['datalayer'] = lambda: datalayer.DbFuncs(self.ctx, self.dal)
+
+        # self['localfs'] = lambda: datalayer.LocalFs(self.ctx, datalayer)
 
     def interactive_shell(self):
         pass

smileyface/config.spec

@@ -11,6 +11,8 @@ remote_game_host = string(max=255, default='')
 remote_game_dir = string(max=255, default='')
 remote_redirect_host = string(max=255, default='')
 
+sqlite_filename = string(max=255, default='smiles.db')
+
 [logging]
 log_file = string(max=255, default='')
 log_level = option('critical', 'error', 'warning', 'info', 'debug', default='info')
@@ -53,3 +55,8 @@ disable_existing_loggers = boolean(default=False)
                 level = option('critical', 'error', 'warning', 'info', 'debug', default='debug')
                 handlers = string_list(max=8, default=list('stderr',))
                 propagate = boolean(default=True)
+
+                [[[db]]]
+                level = option('critical', 'error', 'warning', 'info', 'debug', default='debug')
+                handlers = string_list(max=8, default=list('stderr',))
+                propagate = boolean(default=True)

smileyface/datalayer.py

@@ -1,5 +0,0 @@
-
-class DataLayer:
-    def __init__(self, ctx):
-        self.ctx = ctx
-

smileyface/datalayer/__init__.py

@@ -0,0 +1,2 @@
+from .datalayer import *
+from .db_ops import *

smileyface/datalayer/create_schema.sql

@@ -0,0 +1,18 @@
+drop table if exists file_paks;
+create table file_paks (
+    file_pak_id integer primary key,
+    fullpath varchar(255),
+    filename varchar(255),
+    md5sum varchar(255),
+    filesize int,
+    validated_state varchar(31),
+    validated_at datetime,
+    validated_against varchar(255),
+    remote_src_md5 varchar(255),
+    record_created_at datetime,
+    record_updated_at datetime,
+
+
+
+    unique(filename)
+);

smileyface/datalayer/datalayer.py

@@ -0,0 +1,36 @@
+from smileyface import myutil
+
+import appdirs
+import sqlite3
+import os
+
+
+class DataLayer:
+    def __init__(self, ctx):
+        self.ctx = ctx
+        self._db_conn = None
+
+    @property
+    def db_conn(self):
+        if self._db_conn is None:
+            self._db_conn = self._create_db_connection()
+        return self._db_conn
+
+    def _create_db_connection(self):
+        local_db_filename = self.ctx.config['app']['sqlite_filename']
+        appdir = appdirs.user_data_dir('smileyface')
+        fullpath = os.path.join(appdir, local_db_filename)
+        self.ctx.log['ut4'].info('sqlite3 filename: %s', fullpath)
+
+        myutil.ensure_dir_exists(fullpath)
+
+        db = sqlite3.connect(fullpath)
+        return db
+
+    def commit(self):
+        self.ctx.log['db'].info('commit()')
+        self.db_conn.commit()
+
+
+
+

smileyface/datalayer/db_ops.py

@@ -0,0 +1,166 @@
+from smileyface import myutil
+from smileyface import structs
+
+import app_skellington._util as apputil
+import appdirs
+import datetime
+import os
+import sqlparse
+
+
+class DbFuncs:
+    def __init__(self, ctx, dal):
+        self.ctx = ctx
+        self.dal = dal
+
+    def create_tables(self):
+        sql_filename = apputil.get_asset(__name__, 'create_schema.sql')
+        with open(sql_filename) as fp:
+            contents_sql = fp.read()
+            stmts = sqlparse.split(contents_sql)
+
+        conn = self.dal.db_conn
+        curs = conn.cursor()
+        for stmt in stmts:
+            print('----')
+            print(stmt)
+            curs.execute(stmt)
+
+        self.dal.commit()
+
+    def truncate_tables(self):
+        conn = self.dal.db_conn
+        curs = conn.cursor()
+        sql = '''
+truncate file_paks;
+        '''
+        for stmt in sqlparse.split(sql):
+            curs.execute(stmt)
+        self.dal.commit()
+
+    def insert_filepak_record(self, record):
+
+        # NOTE(MG) datetime parameter logic could be improved and more complete
+        conn = self.dal.db_conn
+        curs = conn.cursor()
+        sql = '''
+insert into file_paks (
+    file_pak_id,
+    fullpath,
+    filename,
+    md5sum,
+    record_created_at,
+    record_updated_at
+) values (
+    ?, ?, ?, ?,
+    datetime('now', 'localtime'),
+    datetime('now', 'localtime')
+)
+on conflict(filename) do update set
+    file_pak_id = excluded.file_pak_id,
+    fullpath = excluded.fullpath,
+    filename = excluded.filename, -- unique key
+    md5sum = excluded.md5sum,
+    --created at does not update
+    record_updated_at = datetime('now', 'localtime')
+                '''
+        args = (
+            record.file_pak_id,
+            record.fullpath,
+            record.filename,
+            record.md5sum
+            # record.record_created_at,
+            # record.record_updated_at
+        )
+        # print(sql)
+        curs.execute(sql, args)
+        self.dal.commit()
+
+    def mark_filepak_validated_state(self, rec_id, validate_state, src, remote_src_md5):
+        # validation data src
+        conn = self.dal.db_conn
+        curs = conn.cursor()
+        sql = '''
+update file_paks
+set
+    validated_state = ?,
+    validated_at = datetime('now', 'localtime'),
+    validated_against = ?,
+    remote_src_md5 = ?
+where
+    file_pak_id = ?
+    
+'''
+        args = (validate_state, src, remote_src_md5, rec_id)
+        curs.execute(sql, args)
+        conn.commit()
+
+    def query_filepak(self, filename):
+        conn = self.dal.db_conn
+        curs = conn.cursor()
+        sql = '''
+select 
+    file_pak_id, fullpath, filename,
+    md5sum, record_created_at, record_updated_at
+from
+    file_paks
+where
+    lower(filename) = lower(?)'''
+        args = (filename,)
+        # print(sql)
+        curs.execute(sql, args)
+        rows = curs.fetchall()
+        output = []
+        for r in rows:
+            filepak = structs.FilePak()
+            print(r)
+            filepak.file_pak_id = r[0]
+            filepak.fullpath = r[1]
+            filepak.filename = r[2]
+            filepak.md5sum = r[3]
+            filepak.record_created_at = r[4]
+            filepak.record_updated_at = r[5]
+            output.append(filepak)
+        if len(output) == 0:
+            return
+        elif len(output) == 1:
+            return output[0]
+        elif len(output) > 1:
+            input('<breakpoint> unexpected two rows returned from db when expecting to be unique')
+            return output
+        return output
+
+
+    def query_invalid_filepaks(self):
+        conn = self.dal.db_conn
+        curs = conn.cursor()
+        sql = '''
+select 
+    file_pak_id, fullpath, filename,
+    md5sum, 
+    validated_state, validated_at, validated_against,
+    record_created_at, record_updated_at
+from
+    file_paks
+where
+    lower(filename) = lower(?)'''
+        args = (filename,)
+        # print(sql)
+        curs.execute(sql, args)
+        rows = curs.fetchall()
+        output = []
+        for r in rows:
+            filepak = structs.FilePak()
+            print(r)
+            filepak.file_pak_id = r[0]
+            filepak.fullpath = r[1]
+            filepak.filename = r[2]
+            filepak.md5sum = r[3]
+            filepak.record_created_at = r[7]
+            filepak.record_updated_at = r[8]
+            output.append(filepak)
+        return output
+
+
+
+

smileyface/hub_machine.py

@@ -1,10 +1,12 @@
 from .gameconfig_edit import UnrealIniFile, GameIniSpecial
 from ._util import md5sum_file
+from . import myutil
+from . import structs
 
-from app_skellington import _util
 import collections
 import configobj
 import configparser
+import datetime
 import glob
 import os
 import pathlib
@@ -12,13 +14,17 @@ import subprocess
 import sys
 import time
 
+
 class UT4ServerMachine:
-    def __init__(self, ctx):
+    def __init__(self, ctx, datalayer):
         self.ctx = ctx
+        self.datalayer = datalayer
 
         if not self._validate_env_vars():
             sys.exit(1)
 
+
+
     def oneclickdeploy(self):
         self.generate_instance()
         self.upload_redirects()

smileyface/myutil.py

@@ -0,0 +1,18 @@
+import hashlib
+import os
+
+
+def ensure_dir_exists(dirpath):
+    if dirpath is None:
+        return
+    if dirpath == '':
+        return
+    dirpath = os.path.dirname(dirpath)
+    os.makedirs(dirpath, exist_ok=True)
+
+
+def md5_file(filename):
+    with open(filename, 'rb') as fp:
+        data = fp.read()
+        h = hashlib.md5(data).hexdigest()
+        return h

smileyface/scrape_latest/__init__.py

@@ -0,0 +1,3 @@
+from .local_fs import *
+from .scrape_utcc import *
+from .scrape_ut4pugs import *

smileyface/scrape_latest/local_fs.py

@@ -0,0 +1,45 @@
+from smileyface import myutil
+from smileyface import structs
+
+import datetime
+import os
+
+
+class LocalFs:
+    def __init__(self, ctx, datalayer):
+        self.ctx = ctx
+        self.datalayer = datalayer
+
+    def create_db_table(self):
+        self.datalayer.create_tables()
+
+    def load_md5s(self):
+        paks_dir = self.ctx.config['app']['project_dir']
+        maps_dir = os.path.join(paks_dir, 'files', 'maps')
+        print(maps_dir)
+        self._load_md5_one_dir(maps_dir)
+
+        muts_dir = os.path.join(paks_dir, 'files', 'mutators')
+        print(muts_dir)
+        self._load_md5_one_dir(muts_dir)
+
+    def _load_md5_one_dir(self, dirname):
+        for fn in os.listdir(dirname):
+            fullpath = os.path.join(dirname, fn)
+            print(fn)
+            md5sum = myutil.md5_file(fullpath)
+            print(md5sum)
+
+            record = structs.FilePak()
+            record.fullpath = fullpath
+            record.filename = fn
+            record.md5sum = md5sum
+            record.record_created_at = datetime.datetime.now()
+            record.record_updated_at = datetime.datetime.now()
+
+            self.datalayer.insert_filepak_record(record)
+
+    def print_invalid_filepaks(self):
+        filepaks = self.datalayer.query_invalid_filepaks()
+        for pak in filepaks:
+            print(pak)

smileyface/scrape_latest/scrape_ut4pugs.py

@@ -0,0 +1,65 @@
+import selenium
+import selenium.webdriver
+
+URL_MUTATORS = 'https://ut4pugs.us/redirect-mutators'
+URL_MAPS = 'https://ut4pugs.us/redirect-mutators'
+
+
+class ScrapeUt4Pugs:
+    def __init__(self, ctx, datalayer):
+        self.ctx = ctx
+        self.datalayer = datalayer
+        self._browser = None
+
+    @property
+    def browser(self):
+        if not self._browser:
+            self._browser = selenium.webdriver.Firefox()
+        return self._browser
+
+    def check_ut4pugs_for_latest(self):
+        self.browser.get(URL_MUTATORS)
+        self._check_pak_md5sums()
+
+        self.browser.get(URL_MAPS)
+        self._check_pak_md5sums()
+
+    def _check_pak_md5sums(self):
+        tbl_of_mutators = self.browser.find_element_by_id('myTable')
+        print(tbl_of_mutators)
+
+        mut_rows = tbl_of_mutators.find_elements_by_xpath('tbody/tr')
+        for r in mut_rows:
+            mut_cols = r.find_elements_by_xpath('td')
+            if len(mut_cols) != 3:
+                input('<breakpoint> at unexpected columns for mutator. received {}'.format(len(mut_cols)))
+            mut_file = mut_cols[0]
+            mut_md5 = mut_cols[1]
+            mut_ini_line = mut_cols[2]
+
+            print(mut_file.text)
+            print(mut_md5.text)
+            # print(mut_ini_line.text)
+
+            local_file = self.datalayer.query_filepak(mut_file.text)
+            print(local_file)
+            if not local_file:
+                self.ctx.log['ut4'].warn('pak not found locally: %s', mut_file.text)
+                continue
+
+            local_md5 = local_file.md5sum
+            remote_md5 = mut_md5.text
+
+            if local_md5 != remote_md5:
+                input('<breakpoint> as mismatching md5!')
+                print('local: ', local_md5)
+                print('remote: ', remote_md5)
+                self.datalayer.mark_filepak_validated_state(local_file.file_pak_id, 'mismatch', 'ut4pugs', remote_md5)
+            else:
+                input('<breakpoint> as matching md5! good job')
+                self.datalayer.mark_filepak_validated_state(local_file.file_pak_id, 'valid', 'ut4pugs', None)
+
+            # print(r)
+        print('xxx')
+        pass
+        pass

smileyface/scrape_latest/scrape_utcc.py

@@ -0,0 +1,24 @@
+import selenium
+import selenium.webdriver
+
+URL_CARDS_LIST = 'https://utcc.unrealpugs.com/content'
+
+
+class ScrapeUtcc:
+    def __init__(self, ctx, datalayer):
+        self.ctx = ctx
+        self.datalayer = datalayer
+        self._browser = None
+
+    @property
+    def browser(self):
+        if not self._browser:
+            self._browser = selenium.webdriver.Firefox()
+        return self._browser
+
+    def check_ut4cc_for_latest(self):
+        self._scrape_list_of_content_cards()
+
+    def _scrape_list_of_content_cards(self):
+        self.browser.get(URL_CARDS_LIST)
+

smileyface/structs.py

@@ -0,0 +1,11 @@
+import datetime
+
+
+class FilePak:
+    def __init__(self):
+        self.file_pak_id = None
+        self.fullpath = None
+        self.filename = None
+        self.md5sum = None
+        self.record_created_at = None
+        self.record_updated_at = None