slight rework cfg module, moving file validation and logic into setters. slight improvement on logging and tests

This commit is contained in:
Mathew Guest 2020-07-18 23:02:44 -06:00
parent 64ee90066d
commit 4e083d6a3d
14 changed files with 352 additions and 173 deletions

@ -23,14 +23,19 @@ _log_fmt = '%(levelname)-7s:%(message)s'
_logger_name = 'app_skellington' _logger_name = 'app_skellington'
_bootstrap_logger = logging.getLogger(_logger_name) _bootstrap_logger = logging.getLogger(_logger_name)
# Logging is manually switched on via environment variable:
if os.environ.get('APP_SKELLINGTON_DEBUG', None): # NOTE(MG) This is done twice: once when app_skellington
# module is imported via _bootstrap.py and again if logging
# configuration is reloaded. This catches APPSKELLINGTON_DEBUG
# environment variable the first time, as app_skellington module
# is imported. See cfg.py
if os.environ.get('APPSKELLINGTON_DEBUG', None):
_bootstrap_logger.setLevel(logging.DEBUG) # Don't filter any log levels _bootstrap_logger.setLevel(logging.DEBUG) # Don't filter any log levels
fmt = logging.Formatter(_log_fmt) fmt = logging.Formatter(_log_fmt)
handler = logging.StreamHandler() handler = logging.StreamHandler()
handler.setFormatter(fmt) handler.setFormatter(fmt)
_bootstrap_logger.addHandler(handler) _bootstrap_logger.addHandler(handler)
_bootstrap_logger.debug('debug log enabled: APP_SKELLINGTON_DEBUG set in environment variables') _bootstrap_logger.debug('log - APPSKELLINGTON_DEBUG set in environment: Enabling verbose logging.')
# Logging is by default off, excepting CRITICAL # Logging is by default off, excepting CRITICAL
else: else:

@ -48,20 +48,19 @@ class ApplicationContainer:
def __init__( def __init__(
self, self,
configspec_filepath=None, configspec_filepath=None,
config_filepath=None, configini_filepath=None,
*args, **kwargs *args, **kwargs
): ):
# Instantiate root application context (container for globals) # Instantiate root application context (container for globals)
if configspec_filepath is None: # if configspec_filepath is None:
configspec_filepath = self._get_configspec_filepath() # configspec_filepath = self._get_configspec_filepath()
self.appname = kwargs.get('appname') or DEFAULT_APP_NAME self.appname = kwargs.get('appname') or DEFAULT_APP_NAME
self.appauthor = kwargs.get('appauthor') or DEFAULT_APP_AUTHOR self.appauthor = kwargs.get('appauthor') or DEFAULT_APP_AUTHOR
self._dependencies = {} self._dependencies = {}
config = cfg.Config(configspec_filepath) config = cfg.Config(configspec_filepath, configini_filepath)
config.load_config_from_file(config_filepath)
logger = log.LoggingLayer(self.appname, self.appauthor) logger = log.LoggingLayer(self.appname, self.appauthor)

@ -1,3 +1,9 @@
# The cfg module provides Config, the app_skellington service for interfacing
# config.ini files, environment variables (*todo), and
# application-wide configuration. The underlying mechanism is built on top of
# ConfigObj module and it's recommended to use config.spec files to define
# your available configuration of the relevant application.
from . import _util from . import _util
from ._bootstrap import _bootstrap_logger from ._bootstrap import _bootstrap_logger
@ -18,15 +24,99 @@ class Config:
'allow_options_beyond_spec': True, 'allow_options_beyond_spec': True,
} }
def __init__(self, configspec_filepath=None, capabilities=None): def __init__(
self.config_obj = None # Reference to configobj.ConfigObj() self,
self._config_filepaths = [] configspec_filepath=None,
configini_filepath=None,
capabilities=None
):
self._config_obj = None # must be type configobj.ConfigObj()
self._configini_data = None
self._configini_filepath = None
self._configspec_data = None
self._configspec_filepath = None self._configspec_filepath = None
# self.configspec_filepath = configspec_filepath self._has_accessed_internal_configobj = True
self._has_changed_internally = True
self._capability_enforce_strict_spec_validation = False
# Register the files and parse provided config.spec and config.ini
self.configspec_filepath = configspec_filepath
self.configini_filepath = configini_filepath
# NOTE(MG) Setters above trigger load_config()
@property
def config_obj(self):
self._has_accessed_internal_configobj = True
return self._config_obj
# config_filepath:
###########################################################################
@property
def configini_filepath(self):
"""
Config.ini filepath of site-specified configuration settings. File
stored on user machine. Reloads config when set.
"""
return self._configini_filepath
@configini_filepath.setter
def configini_filepath(self, value):
self._configini_filepath = value
self._has_changed_internally = True
self.load_config()
# configspec_filepath:
###########################################################################
@property
def configspec_filepath(self):
"""
Config.spec filepath of site-specified configuration settings. File
stored in module directory. Reloads config when set.
"""
return self._configspec_filepath
@configspec_filepath.setter
def configspec_filepath(self, filepath):
# Check if exists as file (at least seems to):
# if not _util.does_file_exist(filepath):
# _bootstrap_logger.error(
# 'failed to set config.spec: file not found '
# '(%s)', filepath)
# raise Exception
if filepath is None:
_bootstrap_logger.debug(
'cfg - clearing configspec'
)
self._configspec_filepath = None
self._configspec_data = None
self._has_changed_internally = True
self.load_config()
return
try:
with open(filepath) as fp:
data = fp.read()
self._configspec_filepath = filepath
self._configspec_data = data
self._has_changed_internally = True
_bootstrap_logger.debug(
'cfg - set configspec and read contents: %s',
filepath
)
self.load_config()
return
except OSError as ex:
_bootstrap_logger.critical(
'cfg - failed to find config.spec: file not found (%s)',
filepath
)
raise OSError('failed to read provided config.spec file')
self.load_config()
def __contains__(self, key): def __contains__(self, key):
try: try:
has_item = key in self.config_obj has_item = key in self._config_obj
return has_item return has_item
except KeyError as ex: except KeyError as ex:
pass pass
@ -46,9 +136,13 @@ class Config:
Returns the value of the configuration item identified by <key>. Returns the value of the configuration item identified by <key>.
""" """
try: try:
return self.config_obj[key].dict() v = self._config_obj[key]
if isinstance(v, str):
return v
else:
# return self._config_obj[key].dict()
return self._config_obj[key]
except KeyError as ex: except KeyError as ex:
# raise ConfigurationItemNotFoundError()
raise raise
def __setitem__(self, key, value): def __setitem__(self, key, value):
@ -56,135 +150,103 @@ class Config:
Assigns the value of the configuration item Assigns the value of the configuration item
identified by <key> as <value>. identified by <key> as <value>.
""" """
self[key] = value self._config_obj[key] = value
@property def load_config(
def config_filepath(self, idx=0): self, configspec_filepath=None, configini_filepath=None
""" ):
Returns the config filepath (optionally specified by index # Set new arguments if were passed in:
when using multiple config files). if configspec_filepath is not None:
""" self.configspec_filepath = configspec_filepath
assert idx>=0, 'invalid idx argument: index must be greater than 0' if configini_filepath is not None:
if len(self._config_filepaths) > 0: self.configini_filepath = configini_filepath
try:
return self._config_filepaths[idx]
except ValueError as ex:
return
@config_filepath.setter # Load and validate configuration if possible:
def config_filepath(self, value, idx=0): self._load_config_files()
"""
Assigns <value> as the config filepath (optionally specified by index
when using multiple config files).
"""
assert idx>=0, 'invalid idx argument: index must be greater than 0'
self._config_filepaths[idx] = value
@property
def configspec_filepath(self):
return self._configspec_filepath
@configspec_filepath.setter
def configspec_filepath(self, filepath):
if _util.does_file_exist(filepath):
self._configspec_filepath = filepath
else:
_bootstrap_logger.error(
'failed to set config.spec: file not found '
'(%s)', filepath)
def load_config_from_file(self, config_filepath):
"""
Loads configuration settings from file, overwritting all configuration.
"""
# Record all config.ini files passed in
if config_filepath not in self._config_filepaths:
self._config_filepaths.append(config_filepath)
# Check for config.spec
if self.configspec_filepath: if self.configspec_filepath:
_bootstrap_logger.info('using config.spec: %s', self.configspec_filepath) rc = self._validate_config_against_spec()
else: if not rc:
_bootstrap_logger.info('config.spec not defined') if self._capability_enforce_strict_spec_validation:
_bootstrap_logger.info('using config file: %s', config_filepath) raise RuntimeError('Failed to validate config.ini against spec.')
return False
return True
# Pre-check for config.ini existence def _load_config_files(self):
if _util.does_file_exist(config_filepath): config_spec = self.configspec_filepath
_bootstrap_logger.info('existing config file found') config_ini = self.configini_filepath
else:
_bootstrap_logger.info('no config file found: using defaults')
# interpolation='template' changes config file variable replacement to # interpolation='template' changes config file variable replacement to
# use the form $var instead of %(var)s, which is useful to enable # use the form $var instead of %(var)s, which is useful to enable
# literal %(text)s values in the config. # literal %(text)s values in the config.
try: try:
configspec_filepath = self.configspec_filepath self._config_obj = configobj.ConfigObj(
if configspec_filepath: infile=config_ini,
self.config_obj = configobj.ConfigObj( # options
config_filepath, configspec=config_spec,
configspec=configspec_filepath, # encoding
interpolation='template' interpolation='template'
# raise_errors
) )
else: _bootstrap_logger.debug(
self.config_obj = configobj.ConfigObj( 'cfg - Parsed configuration. config.spec = %s, config.ini = %s',
config_filepath, config_spec, config_ini
# configspec=configspec_filepath,
interpolation='template'
) )
return True
except configobj.ParseError as ex: except configobj.ParseError as ex:
msg = 'failed to load config: error in config.spec configuration: {}'.format(config_filepath) msg = 'cfg - Failed to load config: error in config.spec configuration: {}'.format(config_filepath)
_bootstrap_logger.error(msg) _bootstrap_logger.error(msg)
_util.eprint(msg)
return False return False
except OSError as ex: except OSError as ex:
msg = 'failed to load config: config.spec file not found' msg = 'cfg - Failed to load config: config.spec file not found.'
_bootstrap_logger.error(msg) _bootstrap_logger.error(msg)
_util.eprint(msg)
return False return False
except Exception as ex:
print(ex)
def _validate_config_against_spec(self):
config_spec = self.configspec_filepath
config_ini = self.configini_filepath
# Hack the configobj module to alter the interpolation for validate.py: # Hack the configobj module to alter the interpolation for validate.py:
configobj.DEFAULT_INTERPOLATION = 'template' configobj.DEFAULT_INTERPOLATION = 'template'
self.config_obj.filename = config_filepath
if self.configspec_filepath:
# Validate config.ini against config.spec # Validate config.ini against config.spec
try: try:
_bootstrap_logger.info('validating config file against spec') _bootstrap_logger.info('cfg - Validating config file against spec')
val = validate.Validator() val = validate.Validator()
test_results = self.config_obj.validate( assert isinstance(self._config_obj, configobj.ConfigObj), 'expecting configobj.ConfigObj, received %s' % type(self._config_obj)
# NOTE(MG) copy below instructs configobj to use defaults from spec file
test_results = self._config_obj.validate(
val, copy=True, preserve_errors=True val, copy=True, preserve_errors=True
) )
if test_results is True: if test_results is True:
_bootstrap_logger.info( _bootstrap_logger.info(
'application configuration file passed validation. input = %s, validation spec = %s', 'cfg- Successfully validated configuration against spec. input = %s, validation spec = %s',
config_filepath, configspec_filepath config_ini, config_spec
)
return True
elif test_results is False:
_bootstrap_logger.debug(
'cfg - Potentially discovered invalid config.spec'
) )
else: else:
_bootstrap_logger.critical('config file failed validation') self._validate_parse_errors(test_results)
for (section_list, key, rslt) in configobj.flatten_errors(self.config_obj, test_results):
_bootstrap_logger.critical('config error info: %s %s %s', section_list, key, rslt)
if key is not None:
_bootstrap_logger.critical('config failed validation: [%s].%s appears invalid. msg = %s', '.'.join(section_list), key, rslt)
else:
_bootstrap_logger.critical("config failed validation: missing section, name = '%s'. msg = %s", '.'.join(section_list), rslt)
return False return False
except ValueError as ex: except ValueError as ex:
_bootstrap_logger.error('failed validating configspec') _bootstrap_logger.error('cfg - Failed while validating config against spec. ')
return False return False
# Create the config file if it doesn't exist def _validate_parse_errors(self, test_results):
# if not _util.does_file_exist(config_filepath): _bootstrap_logger.critical('cfg - Config file failed validation.')
if True: for (section_list, key, rslt) in configobj.flatten_errors(self._config_obj, test_results):
_bootstrap_logger.info('writing new config file: %s', config_filepath) _bootstrap_logger.critical('cfg - Config error info: %s %s %s', section_list, key, rslt)
dirname = os.path.dirname(config_filepath) if key is not None:
_util.ensure_dir_exists(dirname) _bootstrap_logger.critical('cfg - Config failed validation: [%s].%s appears invalid. msg = %s', '.'.join(section_list), key, rslt)
self.config_obj.write() else:
_bootstrap_logger.critical("cfg - Config failed validation: missing section, name = '%s'. msg = %s", '.'.join(section_list), rslt)
_bootstrap_logger.info('done loading config file')
return True
def print_config(self): def print_config(self):
""" """
@ -192,16 +254,13 @@ class Config:
""" """
print('config:') print('config:')
self.config_obj.walk(print) self._config_obj.walk(print)
for section in self.config_obj.sections: for section in self._config_obj.sections:
print(section) print(section)
for key in self.config_obj[section]: for key in self._config_obj[section]:
print(' ', self.config_obj[section][key]) print(' ', self._config_obj[section][key])
class EnvironmentVariables: class EnvironmentVariables:
def __init__(self): def __init__(self):
raise NotImplementedError raise NotImplementedError
class ConfigurationItemNotFoundError(Exception):
pass

@ -153,7 +153,8 @@ class CommandTree:
key, key,
help=helptext, help=helptext,
nargs='?', nargs='?',
default=param.default) default=param.default
)
else: else:
helptext = 'required' helptext = 'required'
self.root_parser.add_argument( self.root_parser.add_argument(
@ -219,20 +220,20 @@ class CommandTree:
def run_command(self, args=None): def run_command(self, args=None):
args, unk, success = self.parse(args) args, unk, success = self.parse(args)
if not success: if not success:
_bootstrap_logger.info('SystemExit: Perhaps user invoked --help') _bootstrap_logger.info('cli - SystemExit: Perhaps user invoked --help')
return return
if args is False and unk is False: if args is False and unk is False:
_bootstrap_logger.error('failed parsing args') _bootstrap_logger.error('failed parsing args')
return False return False
_bootstrap_logger.info('received args from shell: %s', args) _bootstrap_logger.info('cli - received args from shell: %s', args)
args = vars(args) args = vars(args)
cmd = self._lookup_command(args) cmd = self._lookup_command(args)
if cmd is None: if cmd is None:
print('cmd is None') print('cmd is None')
_bootstrap_logger.error('failed to find command') _bootstrap_logger.error('cli - failed to find command')
return False return False
return self._invoke_command(cmd, args) return self._invoke_command(cmd, args)
@ -265,10 +266,10 @@ class CommandTree:
input('<broken>') input('<broken>')
val = args.get(argparse_param) val = args.get(argparse_param)
_bootstrap_logger.debug('argparse command is \'{}\' = {}'.format(argparse_param, val)) _bootstrap_logger.debug('cli - argparse command is \'{}\' = {}'.format(argparse_param, val))
lookup = submenu.entries.get(val) lookup = submenu.entries.get(val)
_bootstrap_logger.debug('lookup, entries[{}] = {}'.format(val, lookup)) _bootstrap_logger.debug('cli - lookup, entries[{}] = {}'.format(val, lookup))
# print(submenu.entries) # print(submenu.entries)
# pop value # pop value
@ -295,8 +296,8 @@ class CommandTree:
if param.name in args: if param.name in args:
func_args.append(args[param.name]) func_args.append(args[param.name])
_bootstrap_logger.info('function: %s', func) _bootstrap_logger.info('cli - function: %s', func)
_bootstrap_logger.info('function args: %s', func_args) _bootstrap_logger.info('cli - function args: %s', func_args)
return command_to_be_invoked(*func_args) return command_to_be_invoked(*func_args)
def _get_subparser(self): def _get_subparser(self):
@ -394,12 +395,14 @@ class SubMenu:
key, key,
help=helptext, help=helptext,
nargs='?', nargs='?',
default=param.default) default=param.default
)
else: else:
helptext = 'required' helptext = 'required'
child_node.add_argument( child_node.add_argument(
key, key,
help=helptext) help=helptext
)
# # Wrapper function that instantiates an object and runs a method # # Wrapper function that instantiates an object and runs a method
# # on-demand. The object is created, injected with necessary # # on-demand. The object is created, injected with necessary
@ -419,7 +422,7 @@ class SubMenu:
registered_name = '{}.{}'.format( registered_name = '{}.{}'.format(
self.submenu_path, self.submenu_path,
cmd_name) cmd_name)
_bootstrap_logger.info('registered command: %s', registered_name) _bootstrap_logger.info('cli - registered command: %s', registered_name)
self.entries[cmd_name] = cmd self.entries[cmd_name] = cmd
def create_submenu( def create_submenu(
@ -471,7 +474,7 @@ class SubMenu:
submenu.submenu_path = '{}.{}'.format(self.submenu_path, cmd_entry_name) submenu.submenu_path = '{}.{}'.format(self.submenu_path, cmd_entry_name)
submenu_name = submenu.submenu_path submenu_name = submenu.submenu_path
_bootstrap_logger.info('registered submenu: %s', submenu_name) _bootstrap_logger.info('cli - registered submenu: %s', submenu_name)
self.entries[cmd_entry_name] = submenu self.entries[cmd_entry_name] = submenu
return submenu return submenu

@ -70,16 +70,16 @@ class LoggingLayer:
noise for typical operation. noise for typical operation.
""" """
if config_dict is None: if config_dict is None:
_bootstrap_logger.debug('No application logging configuration provided. Using default') _bootstrap_logger.debug('log - No application logging configuration provided. Using default')
config_dict = DEFAULT_LOG_SETTINGS config_dict = DEFAULT_LOG_SETTINGS
self.transform_config(config_dict) self.transform_config(config_dict)
try: try:
# TODO(MG) switch to pretty-print, as it'd be more human readable # TODO(MG) switch to pretty-print, as it'd be more human readable
_bootstrap_logger.debug('Log configuration: %s', config_dict) _bootstrap_logger.debug('log - Log configuration: %s', config_dict)
logging.config.dictConfig(config_dict) logging.config.dictConfig(config_dict)
_bootstrap_logger.debug('Configured all logging') _bootstrap_logger.debug('log - Configured all logging')
except Exception as ex: except Exception as ex:
print('unable to configure logging:', ex, type(ex)) print('unable to configure logging:', ex, type(ex))
@ -127,7 +127,13 @@ class LoggingLayer:
config_dict['handlers']['file']['filename'] = log_filepath config_dict['handlers']['file']['filename'] = log_filepath
def _add_own_logconfig(self, config_dict): def _add_own_logconfig(self, config_dict):
if os.environ.get('APP_SKELLINGTON_DEBUG', None): # NOTE(MG) This is done twice: once when app_skellington
# module is imported again if logging configuration is
# reloaded. This catches APPSKELLINGTON_DEBUG environment
# variable the second time, when it's being reloaded as a
# logging configuration is read from config file.
# See _bootstrap.py
if os.environ.get('APPSKELLINGTON_DEBUG', None):
if 'app_skellington' not in config_dict['loggers']: if 'app_skellington' not in config_dict['loggers']:
config_dict['loggers']['app_skellington'] = { config_dict['loggers']['app_skellington'] = {
'level': 'debug', 'propagate': 'false' 'level': 'debug', 'propagate': 'false'

18
tests/README.md Normal file

@ -0,0 +1,18 @@
Software tests for app_skellington
==================================
Contained is the early stages of unit testing for app_skellington framework.
Usage
-----
Run all tests (cwd is testing directory):
pytest .
Run test by keyword:
pytest -k "<test keyword>" .
Run test by directory:
pytest <dirname>

0
tests/cfg/__init__.py Normal file

@ -0,0 +1,5 @@
root_option = root_option_val
[app]
sub_option = sub_option_val

@ -0,0 +1,5 @@
root_option = string(max=255, default='def_string')
[app]
sub_option = string(max=255, default='def_sub')

@ -0,0 +1,5 @@
root_option = root_option_val
[app]
sub_option = sub_option_val

@ -0,0 +1,6 @@
root_option = string(max=255, default='def_string')
int_option = integer(min=0, max=100)
[app]
sub_option = string(max=255, default='def_sub')

@ -0,0 +1,2 @@
root_option = invalid(max=255, default='def_string')

97
tests/cfg/test_cfg.py Normal file

@ -0,0 +1,97 @@
from app_skellington.cfg import Config
from app_skellington import _util
import pytest
@pytest.fixture
def sample_configspec_filepath():
return _util.get_asset(__name__, 'sample_config.spec')
@pytest.fixture
def sample_configini_filepath():
return _util.get_asset(__name__, 'sample_config.ini')
@pytest.fixture
def sample_full_configspec_filepath():
return _util.get_asset(__name__, 'sample_config_full.spec')
@pytest.fixture
def sample_full_configini_filepath():
return _util.get_asset(__name__, 'sample_config_full.ini')
@pytest.fixture
def sample_invalid_configspec_filepath():
return _util.get_asset(__name__, 'sample_config_invalid.spec')
class TestConfig_e2e:
def test_allows_reading_ini_and_no_spec(
self, sample_configini_filepath
):
cfg = Config(
configini_filepath=sample_configini_filepath
)
assert cfg['root_option'] == 'root_option_val', 'expecting default from config.spec (didnt get)'
assert cfg['app']['sub_option'] == 'sub_option_val', 'expecting default for sub option'
def test_allows_reading_spec_and_no_ini(
self, sample_configspec_filepath
):
cfg = Config(
configspec_filepath=sample_configspec_filepath
)
assert cfg['root_option'] == 'def_string', 'expecting default from config.spec (didnt get)'
# NOTE(MG) Changed the functionality to not do it this way.
# def test_constructor_fails_with_invalid_spec(
# self, sample_invalid_configspec_filepath
# ):
# with pytest.raises(Exception):
# cfg = Config(
# configspec_filepath=sample_invalid_configspec_filepath
# )
def test_allows_options_beyond_spec(
self, sample_configspec_filepath
):
cfg = Config(
configspec_filepath=sample_configspec_filepath
)
cfg['foo'] = 'test my value'
assert cfg['foo'] == 'test my value'
cfg['app']['bar'] = 'another value'
assert cfg['app']['bar'] == 'another value'
# def test_can_read_config_file_mutiple_times(self):
# pass
def test_can_override_config_file_manually(
self, sample_configini_filepath
):
cfg = Config(
configini_filepath=sample_configini_filepath
)
cfg['root_option'] = 'newval'
assert cfg['root_option'] == 'newval'
cfg['app']['sub_option'] = 'another_new_val'
assert cfg['app']['sub_option'] == 'another_new_val', 'expecting default for sub option'
def test_can_set_option_without_config(self):
cfg = Config()
cfg['foo'] = 'test my value'
assert cfg['foo'] == 'test my value'
cfg['app'] = {}
cfg['app']['bar'] = 'another value'
assert cfg['app']['bar'] == 'another value'
def test_uses_spec_as_defaults(
self, sample_configspec_filepath
):
cfg = Config(
configspec_filepath=sample_configspec_filepath
)
assert cfg['root_option'] == 'def_string', 'expecting default from config.spec (didnt get)'
assert cfg['app']['sub_option'] == 'def_sub', 'expecting default for sub option'

@ -1,31 +0,0 @@
from app_skellington.cfg import Config
class TestConfig_e2e:
def test_allows_reading_with_no_spec(self):
x = Config()
assert True == False
def test_allows_reading_with_sample_spec(self):
x = Config()
assert True == False
def test_constructor_fails_with_invalid_spec(self):
x = Config()
assert True == False
def test_allows_options_beyond_spec(self):
x = Config()
assert True == False
def test_can_read_config_correctly_from_file(self):
pass
def test_can_read_config_file_mutiple_times(self):
pass
def test_can_override_config_file_manually(self):
pass
def test_can_set_option_without_config(self):
pass