alu/tra-superscript
1
0
Fork 0
mirror of https://github.com/titanscouting/tra-superscript.git synced 2024-12-30 19:39:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

moved config functions into Configuration class,

Browse Source 
simplified exception class by removing error codes,
removed exec_threads from module parameters


Former-commit-id: 545ef765653970e1cdebac692eccd227effb2508
This commit is contained in:
Arthur Lu 2022-02-19 06:19:13 +00:00
parent 524a0a211d
commit b43836899d
7 changed files with 404 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

466
src/config.py
View File

@ -1,244 +1,252 @@
import math
import json import json
from multiprocessing import Pool
import os
from cerberus import Validator
from exceptions import ConfigurationError from exceptions import ConfigurationError
from cerberus import Validator
from data import set_database_config, get_database_config from data import set_database_config, get_database_config
from interface import stderr, stdout, INF, ERR from interface import stderr, stdout, INF, ERR
config_path = "config.json" class Configuration:
sample_json = """ path = None
{ config = {}
"persistent":{
"key":{
"database":"",
"tba":"",
"tra":{
"CLIENT_ID":"",
"CLIENT_SECRET":"",
"url": ""
}
},
"config-preference":"local",
"synchronize-config":false
},
"variable":{
"max-threads":0.5,
"team":"",
"event-delay":false,
"loop-delay":0,
"reportable":true,
"teams":[
], _sample_config = {
"modules":{ "persistent":{
"match":{ "key":{
"tests":{ "database":"",
"balls-blocked":[ "tba":"",
"basic_stats", "tra":{
"historical_analysis", "CLIENT_ID":"",
"regression_linear", "CLIENT_SECRET":"",
"regression_logarithmic", "url": ""
"regression_exponential", }
"regression_polynomial", },
"regression_sigmoidal" "config-preference":"local",
], "synchronize-config":False
"balls-collected":[ },
"basic_stats", "variable":{
"historical_analysis", "event-delay":False,
"regression_linear", "loop-delay":0,
"regression_logarithmic", "competition": "2020ilch",
"regression_exponential", "modules":{
"regression_polynomial", "match":{
"regression_sigmoidal" "tests":{
], "balls-blocked":[
"balls-lower-teleop":[ "basic_stats",
"basic_stats", "historical_analysis",
"historical_analysis", "regression_linear",
"regression_linear", "regression_logarithmic",
"regression_logarithmic", "regression_exponential",
"regression_exponential", "regression_polynomial",
"regression_polynomial", "regression_sigmoidal"
"regression_sigmoidal" ],
], "balls-collected":[
"balls-lower-auto":[ "basic_stats",
"basic_stats", "historical_analysis",
"historical_analysis", "regression_linear",
"regression_linear", "regression_logarithmic",
"regression_logarithmic", "regression_exponential",
"regression_exponential", "regression_polynomial",
"regression_polynomial", "regression_sigmoidal"
"regression_sigmoidal" ],
], "balls-lower-teleop":[
"balls-started":[ "basic_stats",
"basic_stats", "historical_analysis",
"historical_analyss", "regression_linear",
"regression_linear", "regression_logarithmic",
"regression_logarithmic", "regression_exponential",
"regression_exponential", "regression_polynomial",
"regression_polynomial", "regression_sigmoidal"
"regression_sigmoidal" ],
], "balls-lower-auto":[
"balls-upper-teleop":[ "basic_stats",
"basic_stats", "historical_analysis",
"historical_analysis", "regression_linear",
"regression_linear", "regression_logarithmic",
"regression_logarithmic", "regression_exponential",
"regression_exponential", "regression_polynomial",
"regression_polynomial", "regression_sigmoidal"
"regression_sigmoidal" ],
], "balls-started":[
"balls-upper-auto":[ "basic_stats",
"basic_stats", "historical_analyss",
"historical_analysis", "regression_linear",
"regression_linear", "regression_logarithmic",
"regression_logarithmic", "regression_exponential",
"regression_exponential", "regression_polynomial",
"regression_polynomial", "regression_sigmoidal"
"regression_sigmoidal" ],
] "balls-upper-teleop":[
} "basic_stats",
}, "historical_analysis",
"metric":{ "regression_linear",
"tests":{ "regression_logarithmic",
"elo":{ "regression_exponential",
"score":1500, "regression_polynomial",
"N":400, "regression_sigmoidal"
"K":24 ],
}, "balls-upper-auto":[
"gl2":{ "basic_stats",
"score":1500, "historical_analysis",
"rd":250, "regression_linear",
"vol":0.06 "regression_logarithmic",
}, "regression_exponential",
"ts":{ "regression_polynomial",
"mu":25, "regression_sigmoidal"
"sigma":8.33 ]
} }
} },
}, "metric":{
"pit":{ "tests":{
"tests":{ "elo":{
"wheel-mechanism":true, "score":1500,
"low-balls":true, "N":400,
"high-balls":true, "K":24
"wheel-success":true, },
"strategic-focus":true, "gl2":{
"climb-mechanism":true, "score":1500,
"attitude":true "rd":250,
} "vol":0.06
} },
} "ts":{
} "mu":25,
} "sigma":8.33
""" }
}
},
"pit":{
"tests":{
"wheel-mechanism":True,
"low-balls":True,
"high-balls":True,
"wheel-success":True,
"strategic-focus":True,
"climb-mechanism":True,
"attitude":True
}
}
}
}
}
def parse_config_persistent(send, config): _validation_schema = {
v = Validator(load_validation_schema(), allow_unknown = True) "persistent": {
isValidated = v.validate(config) "type": "dict",
"required": True,
"require_all": True,
"schema": {
"key": {
"type": "dict",
"require_all":True,
"schema": {
"database": {"type":"string"},
"tba": {"type": "string"},
"tra": {
"type": "dict",
"require_all": True,
"schema": {
"CLIENT_ID": {"type": "string"},
"CLIENT_SECRET": {"type": "string"},
"url": {"type": "string"}
}
}
}
},
"config-preference": {"type": "string", "required": True},
"synchronize-config": {"type": "boolean", "required": True}
}
}
}
if not isValidated: def __init__(self, path):
raise ConfigurationError(v.errors, 101) self.path = path
self.load_config()
self.validate_config()
apikey = config["persistent"]["key"]["database"] def load_config(self):
tbakey = config["persistent"]["key"]["tba"] try:
preference = config["persistent"]["config-preference"] f = open(self.path, "r")
sync = config["persistent"]["synchronize-config"] self.config.update(json.load(f))
f.close()
except:
self.config = self._sample_config
self.save_config()
f.close()
raise ConfigurationError("could not find config file at <" + self.path + ">, created new sample config file at that path")
return apikey, tbakey, preference, sync def save_config(self):
f = open(self.path, "w+")
def parse_config_variable(send, config): json.dump(self.config, f, ensure_ascii=False, indent=4)
sys_max_threads = os.cpu_count()
try:
cfg_max_threads = config["variable"]["max-threads"]
except:
raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109)
if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 :
alloc_processes = sys_max_threads + cfg_max_threads
elif cfg_max_threads > 0 and cfg_max_threads < 1:
alloc_processes = math.floor(cfg_max_threads * sys_max_threads)
elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads:
alloc_processes = cfg_max_threads
elif cfg_max_threads == 0:
alloc_processes = sys_max_threads
else:
raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110)
try:
exec_threads = Pool(processes = alloc_processes)
except Exception as e:
send(stderr, INF, e)
raise ConfigurationError("unable to start threads", 200)
send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads")
try:
modules = config["variable"]["modules"]
except:
raise ConfigurationError("variable/modules field is invalid or missing", 102)
if modules == None:
raise ConfigurationError("variable/modules field is empty", 106)
send(stdout, INF, "found and loaded competition, match, metrics, pit from config")
return exec_threads, modules
def resolve_config_conflicts(send, client, config, preference, sync):
if sync:
if preference == "local" or preference == "client":
send(stdout, INF, "config-preference set to local/client, loading local config information")
remote_config = get_database_config(client)
if remote_config != config["variable"]:
set_database_config(client, config["variable"])
send(stdout, INF, "database config was different and was updated")
return config
elif preference == "remote" or preference == "database":
send(stdout, INF, "config-preference set to remote/database, loading remote config information")
remote_config= get_database_config(client)
if remote_config != config["variable"]:
config["variable"] = remote_config
if save_config(config_path, config):
raise ConfigurationError("local config was different but could not be updated", 121)
send(stdout, INF, "local config was different and was updated")
return config
else:
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
else:
if preference == "local" or preference == "client":
send(stdout, INF, "config-preference set to local/client, loading local config information")
return config
elif preference == "remote" or preference == "database":
send(stdout, INF, "config-preference set to remote/database, loading database config information")
config["variable"] = get_database_config(client)
return config
else:
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
def load_config(path, config_vector):
try:
f = open(path, "r")
config_vector.update(json.load(f))
f.close() f.close()
return 0
except:
f = open(path, "w")
f.write(sample_json)
f.close()
return 1
def load_validation_schema(): def validate_config(self):
try: v = Validator(self._validation_schema, allow_unknown = True)
with open("validation-schema.json", "r") as f: isValidated = v.validate(self.config)
return json.load(f)
except:
raise FileNotFoundError("Validation schema not found at validation-schema.json")
def save_config(path, config_vector): if not isValidated:
f = open(path, "w+") raise ConfigurationError("config validation error: " + v.errors)
json.dump(config_vector, f, ensure_ascii=False, indent=4)
f.close() def __getattr__(self, name): # simple linear lookup method for common multikey-value paths, TYPE UNSAFE
return 0 if name == "persistent":
return self.config["persistent"]
elif name == "key":
return self.config["persistent"]["key"]
elif name == "database":
# soon to be deprecated
return self.config["persistent"]["key"]["database"]
elif name == "tba":
return self.config["persistent"]["key"]["tba"]
elif name == "tra":
return self.config["persistent"]["key"]["tra"]
elif name == "priority":
return self.config["persistent"]["config-preference"]
elif name == "sync":
return self.config["persistent"]["synchronize-config"]
elif name == "variable":
return self.config["variable"]
elif name == "event_delay":
return self.config["variable"]["event-delay"]
elif name == "loop_delay":
return self.config["variable"]["loop-delay"]
elif name == "competition":
return self.config["variable"]["competition"]
elif name == "modules":
return self.config["variable"]["modules"]
else:
return None
def __getitem__(self, key):
return self.config[key]
def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme
sync = self.sync
priority = self.priority
if sync:
if priority == "local" or priority == "client":
send(stdout, INF, "config-preference set to local/client, loading local config information")
remote_config = get_database_config(client)
if remote_config != self.config["variable"]:
set_database_config(client, self.config["variable"])
send(stdout, INF, "database config was different and was updated")
# no change to config
elif priority == "remote" or priority == "database":
send(stdout, INF, "config-preference set to remote/database, loading remote config information")
remote_config = get_database_config(client)
if remote_config != self.config["variable"]:
self.config["variable"] = remote_config
self.save_config()
# change variable to match remote
send(stdout, INF, "local config was different and was updated")
else:
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
else:
if priority == "local" or priority == "client":
send(stdout, INF, "config-preference set to local/client, loading local config information")
# no change to config
elif priority == "remote" or priority == "database":
send(stdout, INF, "config-preference set to remote/database, loading database config information")
self.config["variable"] = get_database_config(client)
# change variable to match remote without updating local version
else:
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")

141
src/dep.py Normal file
View File

@ -0,0 +1,141 @@
# contains deprecated functions, not to be used unless nessasary!
import json
sample_json = """
{
"persistent":{
"key":{
"database":"",
"tba":"",
"tra":{
"CLIENT_ID":"",
"CLIENT_SECRET":"",
"url": ""
}
},
"config-preference":"local",
"synchronize-config":false
},
"variable":{
"max-threads":0.5,
"team":"",
"event-delay":false,
"loop-delay":0,
"reportable":true,
"teams":[
],
"modules":{
"match":{
"tests":{
"balls-blocked":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-collected":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-lower-teleop":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-lower-auto":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-started":[
"basic_stats",
"historical_analyss",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-upper-teleop":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
],
"balls-upper-auto":[
"basic_stats",
"historical_analysis",
"regression_linear",
"regression_logarithmic",
"regression_exponential",
"regression_polynomial",
"regression_sigmoidal"
]
}
},
"metric":{
"tests":{
"elo":{
"score":1500,
"N":400,
"K":24
},
"gl2":{
"score":1500,
"rd":250,
"vol":0.06
},
"ts":{
"mu":25,
"sigma":8.33
}
}
},
"pit":{
"tests":{
"wheel-mechanism":true,
"low-balls":true,
"high-balls":true,
"wheel-success":true,
"strategic-focus":true,
"climb-mechanism":true,
"attitude":true
}
}
}
}
}
"""
def load_config(path, config_vector):
try:
f = open(path, "r")
config_vector.update(json.load(f))
f.close()
return 0
except:
f = open(path, "w")
f.write(sample_json)
f.close()
return 1

8
src/exceptions.py
View File

@ -1,11 +1,7 @@
class APIError(Exception): class APIError(Exception):
code = None def __init__(self, str):
def __init__(self, str, endpoint):
super().__init__(str) super().__init__(str)
self.endpoint = endpoint
class ConfigurationError (Exception): class ConfigurationError (Exception):
code = None def __init__(self, str):
def __init__(self, str, code):
super().__init__(str) super().__init__(str)
self.code = code

21
src/module.py
View File

@ -22,7 +22,7 @@ class Module(metaclass = abc.ABCMeta):
def validate_config(self, *args, **kwargs): def validate_config(self, *args, **kwargs):
raise NotImplementedError raise NotImplementedError
@abc.abstractmethod @abc.abstractmethod
def run(self, exec_threads, *args, **kwargs): def run(self, *args, **kwargs):
raise NotImplementedError raise NotImplementedError
class Match (Module): class Match (Module):
@ -46,9 +46,9 @@ class Match (Module):
def validate_config(self): def validate_config(self):
return True, "" return True, ""
def run(self, exec_threads): def run(self):
self._load_data() self._load_data()
self._process_data(exec_threads) self._process_data()
self._push_results() self._push_results()
def _load_data(self): def _load_data(self):
@ -85,7 +85,7 @@ class Match (Module):
if test == "regression_sigmoidal": if test == "regression_sigmoidal":
return an.regression(ranges, data, ['sig']) return an.regression(ranges, data, ['sig'])
def _process_data(self, exec_threads): def _process_data(self):
tests = self.config["tests"] tests = self.config["tests"]
data = self.data data = self.data
@ -103,7 +103,6 @@ class Match (Module):
input_vector.append((team, variable, test, data[team][variable])) input_vector.append((team, variable, test, data[team][variable]))
self.data = input_vector self.data = input_vector
#self.results = list(exec_threads.map(self._simplestats, self.data))
self.results = [] self.results = []
for test_var_data in self.data: for test_var_data in self.data:
self.results.append(self._simplestats(test_var_data)) self.results.append(self._simplestats(test_var_data))
@ -164,15 +163,15 @@ class Metric (Module):
def validate_config(self): def validate_config(self):
return True, "" return True, ""
def run(self, exec_threads): def run(self):
self._load_data() self._load_data()
self._process_data(exec_threads) self._process_data()
self._push_results() self._push_results()
def _load_data(self): def _load_data(self):
self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp) self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp)
def _process_data(self, exec_threads): def _process_data(self):
elo_N = self.config["tests"]["elo"]["N"] elo_N = self.config["tests"]["elo"]["N"]
elo_K = self.config["tests"]["elo"]["K"] elo_K = self.config["tests"]["elo"]["K"]
@ -289,15 +288,15 @@ class Pit (Module):
def validate_config(self): def validate_config(self):
return True, "" return True, ""
def run(self, exec_threads): def run(self):
self._load_data() self._load_data()
self._process_data(exec_threads) self._process_data()
self._push_results() self._push_results()
def _load_data(self): def _load_data(self):
self.data = d.load_pit(self.apikey, self.competition) self.data = d.load_pit(self.apikey, self.competition)
def _process_data(self, exec_threads): def _process_data(self):
tests = self.config["tests"] tests = self.config["tests"]
return_vector = {} return_vector = {}
for team in self.data: for team in self.data:

2
src/pull.py
View File

@ -1,7 +1,7 @@
import requests import requests
import json import json
from exceptions import APIError from exceptions import APIError
from config import load_config from dep import load_config
url = "https://titanscouting.epochml.org" url = "https://titanscouting.epochml.org"
config_tra = {} config_tra = {}

36
src/superscript.py
View File

@ -23,6 +23,9 @@ __changelog__ = """changelog:
- config-preference option selects between prioritizing local config and prioritizing database config - config-preference option selects between prioritizing local config and prioritizing database config
- synchronize-config option selects whether to update the non prioritized config with the prioritized one - synchronize-config option selects whether to update the non prioritized config with the prioritized one
- divided config options between persistent ones (keys), and variable ones (everything else) - divided config options between persistent ones (keys), and variable ones (everything else)
- generalized behavior of various core components by collecting loose functions in several dependencies into classes
- module.py contains classes, each one represents a single data analysis routine
- config.py contains the Configuration class, which stores the configuration information and abstracts the getter methods
0.9.3: 0.9.3:
- improved data loading performance by removing redundant PyMongo client creation (120s to 14s) - improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions - passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
@ -152,16 +155,12 @@ __all__ = [
# imports: # imports:
import json import json
from multiprocessing import freeze_support import os, sys, time
import os import pymongo # soon to be deprecated
import pymongo
import sys
import time
import traceback import traceback
import warnings import warnings
import zmq import zmq
import pull from config import Configuration, ConfigurationError
from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError
from data import get_previous_time, set_current_time, check_new_database_matches from data import get_previous_time, set_current_time, check_new_database_matches
from interface import splash, log, ERR, INF, stdout, stderr from interface import splash, log, ERR, INF, stdout, stderr
from module import Match, Metric, Pit from module import Match, Metric, Pit
@ -171,10 +170,6 @@ config_path = "config.json"
def main(send, verbose = False, profile = False, debug = False): def main(send, verbose = False, profile = False, debug = False):
def close_all(): def close_all():
if "exec_threads" in locals():
exec_threads.terminate()
exec_threads.join()
exec_threads.close()
if "client" in locals(): if "client" in locals():
client.close() client.close()
if "f" in locals(): if "f" in locals():
@ -196,14 +191,11 @@ def main(send, verbose = False, profile = False, debug = False):
send(stdout, INF, "current time: " + str(loop_start)) send(stdout, INF, "current time: " + str(loop_start))
config = {} config = Configuration(config_path)
if load_config(config_path, config):
raise ConfigurationError("could not find config at <" + config_path + ">, generating blank config and exiting", 110)
send(stdout, INF, "found and loaded config at <" + config_path + ">") send(stdout, INF, "found and loaded config at <" + config_path + ">")
apikey, tbakey, preference, sync = parse_config_persistent(send, config) apikey, tbakey = config.database, config.tba
send(stdout, INF, "found and loaded database and tba keys") send(stdout, INF, "found and loaded database and tba keys")
@ -213,13 +205,10 @@ def main(send, verbose = False, profile = False, debug = False):
previous_time = get_previous_time(client) previous_time = get_previous_time(client)
send(stdout, INF, "analysis backtimed to: " + str(previous_time)) send(stdout, INF, "analysis backtimed to: " + str(previous_time))
config = resolve_config_conflicts(send, client, config, preference, sync) config.resolve_config_conflicts(send, client)
config_modules, competition = config.modules, config.competition
exec_threads, config_modules = parse_config_variable(send, config)
if 'competition' in config['variable']:
competition = config['variable']['competition']
else:
competition = pull.get_team_competition()
for m in config_modules: for m in config_modules:
if m in modules: if m in modules:
start = time.time() start = time.time()
@ -227,7 +216,7 @@ def main(send, verbose = False, profile = False, debug = False):
valid = current_module.validate_config() valid = current_module.validate_config()
if not valid: if not valid:
continue continue
current_module.run(exec_threads) current_module.run()
send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds") send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds")
if debug: if debug:
f = open(m + ".log", "w+") f = open(m + ".log", "w+")
@ -360,7 +349,6 @@ def restart(pid_path):
if __name__ == "__main__": if __name__ == "__main__":
if sys.platform.startswith("win"): if sys.platform.startswith("win"):
freeze_support()
start(None, verbose = True) start(None, verbose = True)
else: else:

27
src/validation-schema.json
View File

@ -1,27 +0,0 @@
{
"persistent": {
"type": "dict",
"require_all": true,
"schema": {
"key": {
"type": "dict",
"require_all":true,
"schema": {
"database": {"type":"string"},
"tba": {"type": "string"},
"tra": {
"type": "dict",
"require_all": true,
"schema": {
"CLIENT_ID": {"type": "string"},
"CLIENT_SECRET": {"type": "string"},
"url": {"type": "string"}
}
}
}
},
"config-preference": {"type": "string", "required": true},
"synchronize-config": {"type": "boolean", "required": true}
}
}
}