From b43836899d8a0fabda53b636379b4e2e83ca172b Mon Sep 17 00:00:00 2001 From: Arthur Lu Date: Sat, 19 Feb 2022 06:19:13 +0000 Subject: [PATCH] moved config functions into Configuration class, simplified exception class by removing error codes, removed exec_threads from module parameters Former-commit-id: 545ef765653970e1cdebac692eccd227effb2508 --- src/config.py | 466 +++++++++++++++++++------------------ src/dep.py | 141 +++++++++++ src/exceptions.py | 10 +- src/module.py | 21 +- src/pull.py | 2 +- src/superscript.py | 36 +-- src/validation-schema.json | 27 --- 7 files changed, 404 insertions(+), 299 deletions(-) create mode 100644 src/dep.py delete mode 100644 src/validation-schema.json diff --git a/src/config.py b/src/config.py index 332607b..46493a7 100644 --- a/src/config.py +++ b/src/config.py @@ -1,244 +1,252 @@ -import math import json -from multiprocessing import Pool -import os -from cerberus import Validator from exceptions import ConfigurationError +from cerberus import Validator from data import set_database_config, get_database_config from interface import stderr, stdout, INF, ERR -config_path = "config.json" +class Configuration: -sample_json = """ -{ - "persistent":{ - "key":{ - "database":"", - "tba":"", - "tra":{ - "CLIENT_ID":"", - "CLIENT_SECRET":"", - "url": "" - } - }, - "config-preference":"local", - "synchronize-config":false - }, - "variable":{ - "max-threads":0.5, - "team":"", - "event-delay":false, - "loop-delay":0, - "reportable":true, - "teams":[ - - ], - "modules":{ - "match":{ - "tests":{ - "balls-blocked":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-collected":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-lower-teleop":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-lower-auto":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-started":[ - "basic_stats", - "historical_analyss", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-upper-teleop":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ], - "balls-upper-auto":[ - "basic_stats", - "historical_analysis", - "regression_linear", - "regression_logarithmic", - "regression_exponential", - "regression_polynomial", - "regression_sigmoidal" - ] - } - }, - "metric":{ - "tests":{ - "elo":{ - "score":1500, - "N":400, - "K":24 - }, - "gl2":{ - "score":1500, - "rd":250, - "vol":0.06 - }, - "ts":{ - "mu":25, - "sigma":8.33 - } - } - }, - "pit":{ - "tests":{ - "wheel-mechanism":true, - "low-balls":true, - "high-balls":true, - "wheel-success":true, - "strategic-focus":true, - "climb-mechanism":true, - "attitude":true - } - } - } - } -} -""" + path = None + config = {} -def parse_config_persistent(send, config): - v = Validator(load_validation_schema(), allow_unknown = True) - isValidated = v.validate(config) + _sample_config = { + "persistent":{ + "key":{ + "database":"", + "tba":"", + "tra":{ + "CLIENT_ID":"", + "CLIENT_SECRET":"", + "url": "" + } + }, + "config-preference":"local", + "synchronize-config":False + }, + "variable":{ + "event-delay":False, + "loop-delay":0, + "competition": "2020ilch", + "modules":{ + "match":{ + "tests":{ + "balls-blocked":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-collected":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-started":[ + "basic_stats", + "historical_analyss", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ] + } + }, + "metric":{ + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + } + }, + "pit":{ + "tests":{ + "wheel-mechanism":True, + "low-balls":True, + "high-balls":True, + "wheel-success":True, + "strategic-focus":True, + "climb-mechanism":True, + "attitude":True + } + } + } + } + } - if not isValidated: - raise ConfigurationError(v.errors, 101) + _validation_schema = { + "persistent": { + "type": "dict", + "required": True, + "require_all": True, + "schema": { + "key": { + "type": "dict", + "require_all":True, + "schema": { + "database": {"type":"string"}, + "tba": {"type": "string"}, + "tra": { + "type": "dict", + "require_all": True, + "schema": { + "CLIENT_ID": {"type": "string"}, + "CLIENT_SECRET": {"type": "string"}, + "url": {"type": "string"} + } + } + } + }, + "config-preference": {"type": "string", "required": True}, + "synchronize-config": {"type": "boolean", "required": True} + } + } + } - apikey = config["persistent"]["key"]["database"] - tbakey = config["persistent"]["key"]["tba"] - preference = config["persistent"]["config-preference"] - sync = config["persistent"]["synchronize-config"] + def __init__(self, path): + self.path = path + self.load_config() + self.validate_config() - return apikey, tbakey, preference, sync + def load_config(self): + try: + f = open(self.path, "r") + self.config.update(json.load(f)) + f.close() + except: + self.config = self._sample_config + self.save_config() + f.close() + raise ConfigurationError("could not find config file at <" + self.path + ">, created new sample config file at that path") -def parse_config_variable(send, config): - - sys_max_threads = os.cpu_count() - try: - cfg_max_threads = config["variable"]["max-threads"] - except: - raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109) - if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 : - alloc_processes = sys_max_threads + cfg_max_threads - elif cfg_max_threads > 0 and cfg_max_threads < 1: - alloc_processes = math.floor(cfg_max_threads * sys_max_threads) - elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads: - alloc_processes = cfg_max_threads - elif cfg_max_threads == 0: - alloc_processes = sys_max_threads - else: - raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110) - try: - exec_threads = Pool(processes = alloc_processes) - except Exception as e: - send(stderr, INF, e) - raise ConfigurationError("unable to start threads", 200) - send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads") - - try: - modules = config["variable"]["modules"] - except: - raise ConfigurationError("variable/modules field is invalid or missing", 102) - - if modules == None: - raise ConfigurationError("variable/modules field is empty", 106) - - send(stdout, INF, "found and loaded competition, match, metrics, pit from config") - - return exec_threads, modules - -def resolve_config_conflicts(send, client, config, preference, sync): - - if sync: - if preference == "local" or preference == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") - remote_config = get_database_config(client) - if remote_config != config["variable"]: - set_database_config(client, config["variable"]) - send(stdout, INF, "database config was different and was updated") - return config - elif preference == "remote" or preference == "database": - send(stdout, INF, "config-preference set to remote/database, loading remote config information") - remote_config= get_database_config(client) - if remote_config != config["variable"]: - config["variable"] = remote_config - if save_config(config_path, config): - raise ConfigurationError("local config was different but could not be updated", 121) - send(stdout, INF, "local config was different and was updated") - return config - else: - raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) - else: - if preference == "local" or preference == "client": - send(stdout, INF, "config-preference set to local/client, loading local config information") - return config - elif preference == "remote" or preference == "database": - send(stdout, INF, "config-preference set to remote/database, loading database config information") - config["variable"] = get_database_config(client) - return config - else: - raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120) - -def load_config(path, config_vector): - try: - f = open(path, "r") - config_vector.update(json.load(f)) + def save_config(self): + f = open(self.path, "w+") + json.dump(self.config, f, ensure_ascii=False, indent=4) f.close() - return 0 - except: - f = open(path, "w") - f.write(sample_json) - f.close() - return 1 -def load_validation_schema(): - try: - with open("validation-schema.json", "r") as f: - return json.load(f) - except: - raise FileNotFoundError("Validation schema not found at validation-schema.json") + def validate_config(self): + v = Validator(self._validation_schema, allow_unknown = True) + isValidated = v.validate(self.config) -def save_config(path, config_vector): - f = open(path, "w+") - json.dump(config_vector, f, ensure_ascii=False, indent=4) - f.close() - return 0 \ No newline at end of file + if not isValidated: + raise ConfigurationError("config validation error: " + v.errors) + + def __getattr__(self, name): # simple linear lookup method for common multikey-value paths, TYPE UNSAFE + if name == "persistent": + return self.config["persistent"] + elif name == "key": + return self.config["persistent"]["key"] + elif name == "database": + # soon to be deprecated + return self.config["persistent"]["key"]["database"] + elif name == "tba": + return self.config["persistent"]["key"]["tba"] + elif name == "tra": + return self.config["persistent"]["key"]["tra"] + elif name == "priority": + return self.config["persistent"]["config-preference"] + elif name == "sync": + return self.config["persistent"]["synchronize-config"] + elif name == "variable": + return self.config["variable"] + elif name == "event_delay": + return self.config["variable"]["event-delay"] + elif name == "loop_delay": + return self.config["variable"]["loop-delay"] + elif name == "competition": + return self.config["variable"]["competition"] + elif name == "modules": + return self.config["variable"]["modules"] + else: + return None + + def __getitem__(self, key): + return self.config[key] + + def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme + sync = self.sync + priority = self.priority + + if sync: + if priority == "local" or priority == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + remote_config = get_database_config(client) + if remote_config != self.config["variable"]: + set_database_config(client, self.config["variable"]) + send(stdout, INF, "database config was different and was updated") + # no change to config + elif priority == "remote" or priority == "database": + send(stdout, INF, "config-preference set to remote/database, loading remote config information") + remote_config = get_database_config(client) + if remote_config != self.config["variable"]: + self.config["variable"] = remote_config + self.save_config() + # change variable to match remote + send(stdout, INF, "local config was different and was updated") + else: + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"") + else: + if priority == "local" or priority == "client": + send(stdout, INF, "config-preference set to local/client, loading local config information") + # no change to config + elif priority == "remote" or priority == "database": + send(stdout, INF, "config-preference set to remote/database, loading database config information") + self.config["variable"] = get_database_config(client) + # change variable to match remote without updating local version + else: + raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"") \ No newline at end of file diff --git a/src/dep.py b/src/dep.py new file mode 100644 index 0000000..53abb64 --- /dev/null +++ b/src/dep.py @@ -0,0 +1,141 @@ +# contains deprecated functions, not to be used unless nessasary! + +import json + +sample_json = """ +{ + "persistent":{ + "key":{ + "database":"", + "tba":"", + "tra":{ + "CLIENT_ID":"", + "CLIENT_SECRET":"", + "url": "" + } + }, + "config-preference":"local", + "synchronize-config":false + }, + "variable":{ + "max-threads":0.5, + "team":"", + "event-delay":false, + "loop-delay":0, + "reportable":true, + "teams":[ + + ], + "modules":{ + "match":{ + "tests":{ + "balls-blocked":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-collected":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-lower-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-started":[ + "basic_stats", + "historical_analyss", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-teleop":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ], + "balls-upper-auto":[ + "basic_stats", + "historical_analysis", + "regression_linear", + "regression_logarithmic", + "regression_exponential", + "regression_polynomial", + "regression_sigmoidal" + ] + } + }, + "metric":{ + "tests":{ + "elo":{ + "score":1500, + "N":400, + "K":24 + }, + "gl2":{ + "score":1500, + "rd":250, + "vol":0.06 + }, + "ts":{ + "mu":25, + "sigma":8.33 + } + } + }, + "pit":{ + "tests":{ + "wheel-mechanism":true, + "low-balls":true, + "high-balls":true, + "wheel-success":true, + "strategic-focus":true, + "climb-mechanism":true, + "attitude":true + } + } + } + } +} +""" + +def load_config(path, config_vector): + try: + f = open(path, "r") + config_vector.update(json.load(f)) + f.close() + return 0 + except: + f = open(path, "w") + f.write(sample_json) + f.close() + return 1 \ No newline at end of file diff --git a/src/exceptions.py b/src/exceptions.py index 13d4103..64f97dd 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -1,11 +1,7 @@ class APIError(Exception): - code = None - def __init__(self, str, endpoint): + def __init__(self, str): super().__init__(str) - self.endpoint = endpoint class ConfigurationError (Exception): - code = None - def __init__(self, str, code): - super().__init__(str) - self.code = code \ No newline at end of file + def __init__(self, str): + super().__init__(str) \ No newline at end of file diff --git a/src/module.py b/src/module.py index a059e1a..f5d5023 100644 --- a/src/module.py +++ b/src/module.py @@ -22,7 +22,7 @@ class Module(metaclass = abc.ABCMeta): def validate_config(self, *args, **kwargs): raise NotImplementedError @abc.abstractmethod - def run(self, exec_threads, *args, **kwargs): + def run(self, *args, **kwargs): raise NotImplementedError class Match (Module): @@ -46,9 +46,9 @@ class Match (Module): def validate_config(self): return True, "" - def run(self, exec_threads): + def run(self): self._load_data() - self._process_data(exec_threads) + self._process_data() self._push_results() def _load_data(self): @@ -85,7 +85,7 @@ class Match (Module): if test == "regression_sigmoidal": return an.regression(ranges, data, ['sig']) - def _process_data(self, exec_threads): + def _process_data(self): tests = self.config["tests"] data = self.data @@ -103,7 +103,6 @@ class Match (Module): input_vector.append((team, variable, test, data[team][variable])) self.data = input_vector - #self.results = list(exec_threads.map(self._simplestats, self.data)) self.results = [] for test_var_data in self.data: self.results.append(self._simplestats(test_var_data)) @@ -164,15 +163,15 @@ class Metric (Module): def validate_config(self): return True, "" - def run(self, exec_threads): + def run(self): self._load_data() - self._process_data(exec_threads) + self._process_data() self._push_results() def _load_data(self): self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp) - def _process_data(self, exec_threads): + def _process_data(self): elo_N = self.config["tests"]["elo"]["N"] elo_K = self.config["tests"]["elo"]["K"] @@ -289,15 +288,15 @@ class Pit (Module): def validate_config(self): return True, "" - def run(self, exec_threads): + def run(self): self._load_data() - self._process_data(exec_threads) + self._process_data() self._push_results() def _load_data(self): self.data = d.load_pit(self.apikey, self.competition) - def _process_data(self, exec_threads): + def _process_data(self): tests = self.config["tests"] return_vector = {} for team in self.data: diff --git a/src/pull.py b/src/pull.py index 5a95de6..09127b0 100644 --- a/src/pull.py +++ b/src/pull.py @@ -1,7 +1,7 @@ import requests import json from exceptions import APIError -from config import load_config +from dep import load_config url = "https://titanscouting.epochml.org" config_tra = {} diff --git a/src/superscript.py b/src/superscript.py index 970ab52..1cb8bff 100644 --- a/src/superscript.py +++ b/src/superscript.py @@ -23,6 +23,9 @@ __changelog__ = """changelog: - config-preference option selects between prioritizing local config and prioritizing database config - synchronize-config option selects whether to update the non prioritized config with the prioritized one - divided config options between persistent ones (keys), and variable ones (everything else) + - generalized behavior of various core components by collecting loose functions in several dependencies into classes + - module.py contains classes, each one represents a single data analysis routine + - config.py contains the Configuration class, which stores the configuration information and abstracts the getter methods 0.9.3: - improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions @@ -152,16 +155,12 @@ __all__ = [ # imports: import json -from multiprocessing import freeze_support -import os -import pymongo -import sys -import time +import os, sys, time +import pymongo # soon to be deprecated import traceback import warnings import zmq -import pull -from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError +from config import Configuration, ConfigurationError from data import get_previous_time, set_current_time, check_new_database_matches from interface import splash, log, ERR, INF, stdout, stderr from module import Match, Metric, Pit @@ -171,10 +170,6 @@ config_path = "config.json" def main(send, verbose = False, profile = False, debug = False): def close_all(): - if "exec_threads" in locals(): - exec_threads.terminate() - exec_threads.join() - exec_threads.close() if "client" in locals(): client.close() if "f" in locals(): @@ -196,14 +191,11 @@ def main(send, verbose = False, profile = False, debug = False): send(stdout, INF, "current time: " + str(loop_start)) - config = {} - - if load_config(config_path, config): - raise ConfigurationError("could not find config at <" + config_path + ">, generating blank config and exiting", 110) + config = Configuration(config_path) send(stdout, INF, "found and loaded config at <" + config_path + ">") - apikey, tbakey, preference, sync = parse_config_persistent(send, config) + apikey, tbakey = config.database, config.tba send(stdout, INF, "found and loaded database and tba keys") @@ -213,13 +205,10 @@ def main(send, verbose = False, profile = False, debug = False): previous_time = get_previous_time(client) send(stdout, INF, "analysis backtimed to: " + str(previous_time)) - config = resolve_config_conflicts(send, client, config, preference, sync) + config.resolve_config_conflicts(send, client) + + config_modules, competition = config.modules, config.competition - exec_threads, config_modules = parse_config_variable(send, config) - if 'competition' in config['variable']: - competition = config['variable']['competition'] - else: - competition = pull.get_team_competition() for m in config_modules: if m in modules: start = time.time() @@ -227,7 +216,7 @@ def main(send, verbose = False, profile = False, debug = False): valid = current_module.validate_config() if not valid: continue - current_module.run(exec_threads) + current_module.run() send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds") if debug: f = open(m + ".log", "w+") @@ -360,7 +349,6 @@ def restart(pid_path): if __name__ == "__main__": if sys.platform.startswith("win"): - freeze_support() start(None, verbose = True) else: diff --git a/src/validation-schema.json b/src/validation-schema.json deleted file mode 100644 index 3b4c95c..0000000 --- a/src/validation-schema.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "persistent": { - "type": "dict", - "require_all": true, - "schema": { - "key": { - "type": "dict", - "require_all":true, - "schema": { - "database": {"type":"string"}, - "tba": {"type": "string"}, - "tra": { - "type": "dict", - "require_all": true, - "schema": { - "CLIENT_ID": {"type": "string"}, - "CLIENT_SECRET": {"type": "string"}, - "url": {"type": "string"} - } - } - } - }, - "config-preference": {"type": "string", "required": true}, - "synchronize-config": {"type": "boolean", "required": true} - } - } -}