mirror of
https://github.com/titanscouting/tra-superscript.git
synced 2024-12-29 02:59:08 +00:00
moved config functions into Configuration class,
simplified exception class by removing error codes, removed exec_threads from module parameters Former-commit-id: 545ef765653970e1cdebac692eccd227effb2508
This commit is contained in:
parent
524a0a211d
commit
b43836899d
466
src/config.py
466
src/config.py
@ -1,244 +1,252 @@
|
|||||||
import math
|
|
||||||
import json
|
import json
|
||||||
from multiprocessing import Pool
|
|
||||||
import os
|
|
||||||
from cerberus import Validator
|
|
||||||
from exceptions import ConfigurationError
|
from exceptions import ConfigurationError
|
||||||
|
from cerberus import Validator
|
||||||
|
|
||||||
from data import set_database_config, get_database_config
|
from data import set_database_config, get_database_config
|
||||||
from interface import stderr, stdout, INF, ERR
|
from interface import stderr, stdout, INF, ERR
|
||||||
|
|
||||||
config_path = "config.json"
|
class Configuration:
|
||||||
|
|
||||||
sample_json = """
|
path = None
|
||||||
{
|
config = {}
|
||||||
"persistent":{
|
|
||||||
"key":{
|
|
||||||
"database":"",
|
|
||||||
"tba":"",
|
|
||||||
"tra":{
|
|
||||||
"CLIENT_ID":"",
|
|
||||||
"CLIENT_SECRET":"",
|
|
||||||
"url": ""
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"config-preference":"local",
|
|
||||||
"synchronize-config":false
|
|
||||||
},
|
|
||||||
"variable":{
|
|
||||||
"max-threads":0.5,
|
|
||||||
"team":"",
|
|
||||||
"event-delay":false,
|
|
||||||
"loop-delay":0,
|
|
||||||
"reportable":true,
|
|
||||||
"teams":[
|
|
||||||
|
|
||||||
],
|
|
||||||
"modules":{
|
|
||||||
"match":{
|
|
||||||
"tests":{
|
|
||||||
"balls-blocked":[
|
|
||||||
"basic_stats",
|
|
||||||
"historical_analysis",
|
|
||||||
"regression_linear",
|
|
||||||
"regression_logarithmic",
|
|
||||||
"regression_exponential",
|
|
||||||
"regression_polynomial",
|
|
||||||
"regression_sigmoidal"
|
|
||||||
],
|
|
||||||
"balls-collected":[
|
|
||||||
"basic_stats",
|
|
||||||
"historical_analysis",
|
|
||||||
"regression_linear",
|
|
||||||
"regression_logarithmic",
|
|
||||||
"regression_exponential",
|
|
||||||
"regression_polynomial",
|
|
||||||
"regression_sigmoidal"
|
|
||||||
],
|
|
||||||
"balls-lower-teleop":[
|
|
||||||
"basic_stats",
|
|
||||||
"historical_analysis",
|
|
||||||
"regression_linear",
|
|
||||||
"regression_logarithmic",
|
|
||||||
"regression_exponential",
|
|
||||||
"regression_polynomial",
|
|
||||||
"regression_sigmoidal"
|
|
||||||
],
|
|
||||||
"balls-lower-auto":[
|
|
||||||
"basic_stats",
|
|
||||||
"historical_analysis",
|
|
||||||
"regression_linear",
|
|
||||||
"regression_logarithmic",
|
|
||||||
"regression_exponential",
|
|
||||||
"regression_polynomial",
|
|
||||||
"regression_sigmoidal"
|
|
||||||
],
|
|
||||||
"balls-started":[
|
|
||||||
"basic_stats",
|
|
||||||
"historical_analyss",
|
|
||||||
"regression_linear",
|
|
||||||
"regression_logarithmic",
|
|
||||||
"regression_exponential",
|
|
||||||
"regression_polynomial",
|
|
||||||
"regression_sigmoidal"
|
|
||||||
],
|
|
||||||
"balls-upper-teleop":[
|
|
||||||
"basic_stats",
|
|
||||||
"historical_analysis",
|
|
||||||
"regression_linear",
|
|
||||||
"regression_logarithmic",
|
|
||||||
"regression_exponential",
|
|
||||||
"regression_polynomial",
|
|
||||||
"regression_sigmoidal"
|
|
||||||
],
|
|
||||||
"balls-upper-auto":[
|
|
||||||
"basic_stats",
|
|
||||||
"historical_analysis",
|
|
||||||
"regression_linear",
|
|
||||||
"regression_logarithmic",
|
|
||||||
"regression_exponential",
|
|
||||||
"regression_polynomial",
|
|
||||||
"regression_sigmoidal"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"metric":{
|
|
||||||
"tests":{
|
|
||||||
"elo":{
|
|
||||||
"score":1500,
|
|
||||||
"N":400,
|
|
||||||
"K":24
|
|
||||||
},
|
|
||||||
"gl2":{
|
|
||||||
"score":1500,
|
|
||||||
"rd":250,
|
|
||||||
"vol":0.06
|
|
||||||
},
|
|
||||||
"ts":{
|
|
||||||
"mu":25,
|
|
||||||
"sigma":8.33
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"pit":{
|
|
||||||
"tests":{
|
|
||||||
"wheel-mechanism":true,
|
|
||||||
"low-balls":true,
|
|
||||||
"high-balls":true,
|
|
||||||
"wheel-success":true,
|
|
||||||
"strategic-focus":true,
|
|
||||||
"climb-mechanism":true,
|
|
||||||
"attitude":true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
def parse_config_persistent(send, config):
|
_sample_config = {
|
||||||
v = Validator(load_validation_schema(), allow_unknown = True)
|
"persistent":{
|
||||||
isValidated = v.validate(config)
|
"key":{
|
||||||
|
"database":"",
|
||||||
|
"tba":"",
|
||||||
|
"tra":{
|
||||||
|
"CLIENT_ID":"",
|
||||||
|
"CLIENT_SECRET":"",
|
||||||
|
"url": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config-preference":"local",
|
||||||
|
"synchronize-config":False
|
||||||
|
},
|
||||||
|
"variable":{
|
||||||
|
"event-delay":False,
|
||||||
|
"loop-delay":0,
|
||||||
|
"competition": "2020ilch",
|
||||||
|
"modules":{
|
||||||
|
"match":{
|
||||||
|
"tests":{
|
||||||
|
"balls-blocked":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-collected":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-lower-teleop":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-lower-auto":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-started":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analyss",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-upper-teleop":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-upper-auto":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metric":{
|
||||||
|
"tests":{
|
||||||
|
"elo":{
|
||||||
|
"score":1500,
|
||||||
|
"N":400,
|
||||||
|
"K":24
|
||||||
|
},
|
||||||
|
"gl2":{
|
||||||
|
"score":1500,
|
||||||
|
"rd":250,
|
||||||
|
"vol":0.06
|
||||||
|
},
|
||||||
|
"ts":{
|
||||||
|
"mu":25,
|
||||||
|
"sigma":8.33
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pit":{
|
||||||
|
"tests":{
|
||||||
|
"wheel-mechanism":True,
|
||||||
|
"low-balls":True,
|
||||||
|
"high-balls":True,
|
||||||
|
"wheel-success":True,
|
||||||
|
"strategic-focus":True,
|
||||||
|
"climb-mechanism":True,
|
||||||
|
"attitude":True
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if not isValidated:
|
_validation_schema = {
|
||||||
raise ConfigurationError(v.errors, 101)
|
"persistent": {
|
||||||
|
"type": "dict",
|
||||||
|
"required": True,
|
||||||
|
"require_all": True,
|
||||||
|
"schema": {
|
||||||
|
"key": {
|
||||||
|
"type": "dict",
|
||||||
|
"require_all":True,
|
||||||
|
"schema": {
|
||||||
|
"database": {"type":"string"},
|
||||||
|
"tba": {"type": "string"},
|
||||||
|
"tra": {
|
||||||
|
"type": "dict",
|
||||||
|
"require_all": True,
|
||||||
|
"schema": {
|
||||||
|
"CLIENT_ID": {"type": "string"},
|
||||||
|
"CLIENT_SECRET": {"type": "string"},
|
||||||
|
"url": {"type": "string"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config-preference": {"type": "string", "required": True},
|
||||||
|
"synchronize-config": {"type": "boolean", "required": True}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
apikey = config["persistent"]["key"]["database"]
|
def __init__(self, path):
|
||||||
tbakey = config["persistent"]["key"]["tba"]
|
self.path = path
|
||||||
preference = config["persistent"]["config-preference"]
|
self.load_config()
|
||||||
sync = config["persistent"]["synchronize-config"]
|
self.validate_config()
|
||||||
|
|
||||||
return apikey, tbakey, preference, sync
|
def load_config(self):
|
||||||
|
try:
|
||||||
|
f = open(self.path, "r")
|
||||||
|
self.config.update(json.load(f))
|
||||||
|
f.close()
|
||||||
|
except:
|
||||||
|
self.config = self._sample_config
|
||||||
|
self.save_config()
|
||||||
|
f.close()
|
||||||
|
raise ConfigurationError("could not find config file at <" + self.path + ">, created new sample config file at that path")
|
||||||
|
|
||||||
def parse_config_variable(send, config):
|
def save_config(self):
|
||||||
|
f = open(self.path, "w+")
|
||||||
sys_max_threads = os.cpu_count()
|
json.dump(self.config, f, ensure_ascii=False, indent=4)
|
||||||
try:
|
|
||||||
cfg_max_threads = config["variable"]["max-threads"]
|
|
||||||
except:
|
|
||||||
raise ConfigurationError("variable/max-threads field is invalid or missing, refer to documentation for configuration options", 109)
|
|
||||||
if cfg_max_threads > -sys_max_threads and cfg_max_threads < 0 :
|
|
||||||
alloc_processes = sys_max_threads + cfg_max_threads
|
|
||||||
elif cfg_max_threads > 0 and cfg_max_threads < 1:
|
|
||||||
alloc_processes = math.floor(cfg_max_threads * sys_max_threads)
|
|
||||||
elif cfg_max_threads > 1 and cfg_max_threads <= sys_max_threads:
|
|
||||||
alloc_processes = cfg_max_threads
|
|
||||||
elif cfg_max_threads == 0:
|
|
||||||
alloc_processes = sys_max_threads
|
|
||||||
else:
|
|
||||||
raise ConfigurationError("variable/max-threads must be between -" + str(sys_max_threads) + " and " + str(sys_max_threads) + ", but got " + cfg_max_threads, 110)
|
|
||||||
try:
|
|
||||||
exec_threads = Pool(processes = alloc_processes)
|
|
||||||
except Exception as e:
|
|
||||||
send(stderr, INF, e)
|
|
||||||
raise ConfigurationError("unable to start threads", 200)
|
|
||||||
send(stdout, INF, "successfully initialized " + str(alloc_processes) + " threads")
|
|
||||||
|
|
||||||
try:
|
|
||||||
modules = config["variable"]["modules"]
|
|
||||||
except:
|
|
||||||
raise ConfigurationError("variable/modules field is invalid or missing", 102)
|
|
||||||
|
|
||||||
if modules == None:
|
|
||||||
raise ConfigurationError("variable/modules field is empty", 106)
|
|
||||||
|
|
||||||
send(stdout, INF, "found and loaded competition, match, metrics, pit from config")
|
|
||||||
|
|
||||||
return exec_threads, modules
|
|
||||||
|
|
||||||
def resolve_config_conflicts(send, client, config, preference, sync):
|
|
||||||
|
|
||||||
if sync:
|
|
||||||
if preference == "local" or preference == "client":
|
|
||||||
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
|
||||||
remote_config = get_database_config(client)
|
|
||||||
if remote_config != config["variable"]:
|
|
||||||
set_database_config(client, config["variable"])
|
|
||||||
send(stdout, INF, "database config was different and was updated")
|
|
||||||
return config
|
|
||||||
elif preference == "remote" or preference == "database":
|
|
||||||
send(stdout, INF, "config-preference set to remote/database, loading remote config information")
|
|
||||||
remote_config= get_database_config(client)
|
|
||||||
if remote_config != config["variable"]:
|
|
||||||
config["variable"] = remote_config
|
|
||||||
if save_config(config_path, config):
|
|
||||||
raise ConfigurationError("local config was different but could not be updated", 121)
|
|
||||||
send(stdout, INF, "local config was different and was updated")
|
|
||||||
return config
|
|
||||||
else:
|
|
||||||
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
|
|
||||||
else:
|
|
||||||
if preference == "local" or preference == "client":
|
|
||||||
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
|
||||||
return config
|
|
||||||
elif preference == "remote" or preference == "database":
|
|
||||||
send(stdout, INF, "config-preference set to remote/database, loading database config information")
|
|
||||||
config["variable"] = get_database_config(client)
|
|
||||||
return config
|
|
||||||
else:
|
|
||||||
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"", 120)
|
|
||||||
|
|
||||||
def load_config(path, config_vector):
|
|
||||||
try:
|
|
||||||
f = open(path, "r")
|
|
||||||
config_vector.update(json.load(f))
|
|
||||||
f.close()
|
f.close()
|
||||||
return 0
|
|
||||||
except:
|
|
||||||
f = open(path, "w")
|
|
||||||
f.write(sample_json)
|
|
||||||
f.close()
|
|
||||||
return 1
|
|
||||||
|
|
||||||
def load_validation_schema():
|
def validate_config(self):
|
||||||
try:
|
v = Validator(self._validation_schema, allow_unknown = True)
|
||||||
with open("validation-schema.json", "r") as f:
|
isValidated = v.validate(self.config)
|
||||||
return json.load(f)
|
|
||||||
except:
|
|
||||||
raise FileNotFoundError("Validation schema not found at validation-schema.json")
|
|
||||||
|
|
||||||
def save_config(path, config_vector):
|
if not isValidated:
|
||||||
f = open(path, "w+")
|
raise ConfigurationError("config validation error: " + v.errors)
|
||||||
json.dump(config_vector, f, ensure_ascii=False, indent=4)
|
|
||||||
f.close()
|
def __getattr__(self, name): # simple linear lookup method for common multikey-value paths, TYPE UNSAFE
|
||||||
return 0
|
if name == "persistent":
|
||||||
|
return self.config["persistent"]
|
||||||
|
elif name == "key":
|
||||||
|
return self.config["persistent"]["key"]
|
||||||
|
elif name == "database":
|
||||||
|
# soon to be deprecated
|
||||||
|
return self.config["persistent"]["key"]["database"]
|
||||||
|
elif name == "tba":
|
||||||
|
return self.config["persistent"]["key"]["tba"]
|
||||||
|
elif name == "tra":
|
||||||
|
return self.config["persistent"]["key"]["tra"]
|
||||||
|
elif name == "priority":
|
||||||
|
return self.config["persistent"]["config-preference"]
|
||||||
|
elif name == "sync":
|
||||||
|
return self.config["persistent"]["synchronize-config"]
|
||||||
|
elif name == "variable":
|
||||||
|
return self.config["variable"]
|
||||||
|
elif name == "event_delay":
|
||||||
|
return self.config["variable"]["event-delay"]
|
||||||
|
elif name == "loop_delay":
|
||||||
|
return self.config["variable"]["loop-delay"]
|
||||||
|
elif name == "competition":
|
||||||
|
return self.config["variable"]["competition"]
|
||||||
|
elif name == "modules":
|
||||||
|
return self.config["variable"]["modules"]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
return self.config[key]
|
||||||
|
|
||||||
|
def resolve_config_conflicts(self, send, client): # needs improvement with new localization scheme
|
||||||
|
sync = self.sync
|
||||||
|
priority = self.priority
|
||||||
|
|
||||||
|
if sync:
|
||||||
|
if priority == "local" or priority == "client":
|
||||||
|
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
||||||
|
remote_config = get_database_config(client)
|
||||||
|
if remote_config != self.config["variable"]:
|
||||||
|
set_database_config(client, self.config["variable"])
|
||||||
|
send(stdout, INF, "database config was different and was updated")
|
||||||
|
# no change to config
|
||||||
|
elif priority == "remote" or priority == "database":
|
||||||
|
send(stdout, INF, "config-preference set to remote/database, loading remote config information")
|
||||||
|
remote_config = get_database_config(client)
|
||||||
|
if remote_config != self.config["variable"]:
|
||||||
|
self.config["variable"] = remote_config
|
||||||
|
self.save_config()
|
||||||
|
# change variable to match remote
|
||||||
|
send(stdout, INF, "local config was different and was updated")
|
||||||
|
else:
|
||||||
|
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
|
||||||
|
else:
|
||||||
|
if priority == "local" or priority == "client":
|
||||||
|
send(stdout, INF, "config-preference set to local/client, loading local config information")
|
||||||
|
# no change to config
|
||||||
|
elif priority == "remote" or priority == "database":
|
||||||
|
send(stdout, INF, "config-preference set to remote/database, loading database config information")
|
||||||
|
self.config["variable"] = get_database_config(client)
|
||||||
|
# change variable to match remote without updating local version
|
||||||
|
else:
|
||||||
|
raise ConfigurationError("persistent/config-preference field must be \"local\"/\"client\" or \"remote\"/\"database\"")
|
141
src/dep.py
Normal file
141
src/dep.py
Normal file
@ -0,0 +1,141 @@
|
|||||||
|
# contains deprecated functions, not to be used unless nessasary!
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
sample_json = """
|
||||||
|
{
|
||||||
|
"persistent":{
|
||||||
|
"key":{
|
||||||
|
"database":"",
|
||||||
|
"tba":"",
|
||||||
|
"tra":{
|
||||||
|
"CLIENT_ID":"",
|
||||||
|
"CLIENT_SECRET":"",
|
||||||
|
"url": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config-preference":"local",
|
||||||
|
"synchronize-config":false
|
||||||
|
},
|
||||||
|
"variable":{
|
||||||
|
"max-threads":0.5,
|
||||||
|
"team":"",
|
||||||
|
"event-delay":false,
|
||||||
|
"loop-delay":0,
|
||||||
|
"reportable":true,
|
||||||
|
"teams":[
|
||||||
|
|
||||||
|
],
|
||||||
|
"modules":{
|
||||||
|
"match":{
|
||||||
|
"tests":{
|
||||||
|
"balls-blocked":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-collected":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-lower-teleop":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-lower-auto":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-started":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analyss",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-upper-teleop":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
],
|
||||||
|
"balls-upper-auto":[
|
||||||
|
"basic_stats",
|
||||||
|
"historical_analysis",
|
||||||
|
"regression_linear",
|
||||||
|
"regression_logarithmic",
|
||||||
|
"regression_exponential",
|
||||||
|
"regression_polynomial",
|
||||||
|
"regression_sigmoidal"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"metric":{
|
||||||
|
"tests":{
|
||||||
|
"elo":{
|
||||||
|
"score":1500,
|
||||||
|
"N":400,
|
||||||
|
"K":24
|
||||||
|
},
|
||||||
|
"gl2":{
|
||||||
|
"score":1500,
|
||||||
|
"rd":250,
|
||||||
|
"vol":0.06
|
||||||
|
},
|
||||||
|
"ts":{
|
||||||
|
"mu":25,
|
||||||
|
"sigma":8.33
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pit":{
|
||||||
|
"tests":{
|
||||||
|
"wheel-mechanism":true,
|
||||||
|
"low-balls":true,
|
||||||
|
"high-balls":true,
|
||||||
|
"wheel-success":true,
|
||||||
|
"strategic-focus":true,
|
||||||
|
"climb-mechanism":true,
|
||||||
|
"attitude":true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
def load_config(path, config_vector):
|
||||||
|
try:
|
||||||
|
f = open(path, "r")
|
||||||
|
config_vector.update(json.load(f))
|
||||||
|
f.close()
|
||||||
|
return 0
|
||||||
|
except:
|
||||||
|
f = open(path, "w")
|
||||||
|
f.write(sample_json)
|
||||||
|
f.close()
|
||||||
|
return 1
|
@ -1,11 +1,7 @@
|
|||||||
class APIError(Exception):
|
class APIError(Exception):
|
||||||
code = None
|
def __init__(self, str):
|
||||||
def __init__(self, str, endpoint):
|
|
||||||
super().__init__(str)
|
super().__init__(str)
|
||||||
self.endpoint = endpoint
|
|
||||||
|
|
||||||
class ConfigurationError (Exception):
|
class ConfigurationError (Exception):
|
||||||
code = None
|
def __init__(self, str):
|
||||||
def __init__(self, str, code):
|
super().__init__(str)
|
||||||
super().__init__(str)
|
|
||||||
self.code = code
|
|
@ -22,7 +22,7 @@ class Module(metaclass = abc.ABCMeta):
|
|||||||
def validate_config(self, *args, **kwargs):
|
def validate_config(self, *args, **kwargs):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def run(self, exec_threads, *args, **kwargs):
|
def run(self, *args, **kwargs):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
class Match (Module):
|
class Match (Module):
|
||||||
@ -46,9 +46,9 @@ class Match (Module):
|
|||||||
def validate_config(self):
|
def validate_config(self):
|
||||||
return True, ""
|
return True, ""
|
||||||
|
|
||||||
def run(self, exec_threads):
|
def run(self):
|
||||||
self._load_data()
|
self._load_data()
|
||||||
self._process_data(exec_threads)
|
self._process_data()
|
||||||
self._push_results()
|
self._push_results()
|
||||||
|
|
||||||
def _load_data(self):
|
def _load_data(self):
|
||||||
@ -85,7 +85,7 @@ class Match (Module):
|
|||||||
if test == "regression_sigmoidal":
|
if test == "regression_sigmoidal":
|
||||||
return an.regression(ranges, data, ['sig'])
|
return an.regression(ranges, data, ['sig'])
|
||||||
|
|
||||||
def _process_data(self, exec_threads):
|
def _process_data(self):
|
||||||
|
|
||||||
tests = self.config["tests"]
|
tests = self.config["tests"]
|
||||||
data = self.data
|
data = self.data
|
||||||
@ -103,7 +103,6 @@ class Match (Module):
|
|||||||
input_vector.append((team, variable, test, data[team][variable]))
|
input_vector.append((team, variable, test, data[team][variable]))
|
||||||
|
|
||||||
self.data = input_vector
|
self.data = input_vector
|
||||||
#self.results = list(exec_threads.map(self._simplestats, self.data))
|
|
||||||
self.results = []
|
self.results = []
|
||||||
for test_var_data in self.data:
|
for test_var_data in self.data:
|
||||||
self.results.append(self._simplestats(test_var_data))
|
self.results.append(self._simplestats(test_var_data))
|
||||||
@ -164,15 +163,15 @@ class Metric (Module):
|
|||||||
def validate_config(self):
|
def validate_config(self):
|
||||||
return True, ""
|
return True, ""
|
||||||
|
|
||||||
def run(self, exec_threads):
|
def run(self):
|
||||||
self._load_data()
|
self._load_data()
|
||||||
self._process_data(exec_threads)
|
self._process_data()
|
||||||
self._push_results()
|
self._push_results()
|
||||||
|
|
||||||
def _load_data(self):
|
def _load_data(self):
|
||||||
self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp)
|
self.data = d.pull_new_tba_matches(self.tbakey, self.competition, self.timestamp)
|
||||||
|
|
||||||
def _process_data(self, exec_threads):
|
def _process_data(self):
|
||||||
|
|
||||||
elo_N = self.config["tests"]["elo"]["N"]
|
elo_N = self.config["tests"]["elo"]["N"]
|
||||||
elo_K = self.config["tests"]["elo"]["K"]
|
elo_K = self.config["tests"]["elo"]["K"]
|
||||||
@ -289,15 +288,15 @@ class Pit (Module):
|
|||||||
def validate_config(self):
|
def validate_config(self):
|
||||||
return True, ""
|
return True, ""
|
||||||
|
|
||||||
def run(self, exec_threads):
|
def run(self):
|
||||||
self._load_data()
|
self._load_data()
|
||||||
self._process_data(exec_threads)
|
self._process_data()
|
||||||
self._push_results()
|
self._push_results()
|
||||||
|
|
||||||
def _load_data(self):
|
def _load_data(self):
|
||||||
self.data = d.load_pit(self.apikey, self.competition)
|
self.data = d.load_pit(self.apikey, self.competition)
|
||||||
|
|
||||||
def _process_data(self, exec_threads):
|
def _process_data(self):
|
||||||
tests = self.config["tests"]
|
tests = self.config["tests"]
|
||||||
return_vector = {}
|
return_vector = {}
|
||||||
for team in self.data:
|
for team in self.data:
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
import requests
|
import requests
|
||||||
import json
|
import json
|
||||||
from exceptions import APIError
|
from exceptions import APIError
|
||||||
from config import load_config
|
from dep import load_config
|
||||||
|
|
||||||
url = "https://titanscouting.epochml.org"
|
url = "https://titanscouting.epochml.org"
|
||||||
config_tra = {}
|
config_tra = {}
|
||||||
|
@ -23,6 +23,9 @@ __changelog__ = """changelog:
|
|||||||
- config-preference option selects between prioritizing local config and prioritizing database config
|
- config-preference option selects between prioritizing local config and prioritizing database config
|
||||||
- synchronize-config option selects whether to update the non prioritized config with the prioritized one
|
- synchronize-config option selects whether to update the non prioritized config with the prioritized one
|
||||||
- divided config options between persistent ones (keys), and variable ones (everything else)
|
- divided config options between persistent ones (keys), and variable ones (everything else)
|
||||||
|
- generalized behavior of various core components by collecting loose functions in several dependencies into classes
|
||||||
|
- module.py contains classes, each one represents a single data analysis routine
|
||||||
|
- config.py contains the Configuration class, which stores the configuration information and abstracts the getter methods
|
||||||
0.9.3:
|
0.9.3:
|
||||||
- improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
|
- improved data loading performance by removing redundant PyMongo client creation (120s to 14s)
|
||||||
- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
|
- passed singular instance of PyMongo client as standin for apikey parameter in all data.py functions
|
||||||
@ -152,16 +155,12 @@ __all__ = [
|
|||||||
# imports:
|
# imports:
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from multiprocessing import freeze_support
|
import os, sys, time
|
||||||
import os
|
import pymongo # soon to be deprecated
|
||||||
import pymongo
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import traceback
|
import traceback
|
||||||
import warnings
|
import warnings
|
||||||
import zmq
|
import zmq
|
||||||
import pull
|
from config import Configuration, ConfigurationError
|
||||||
from config import parse_config_persistent, parse_config_variable, resolve_config_conflicts, load_config, save_config, ConfigurationError
|
|
||||||
from data import get_previous_time, set_current_time, check_new_database_matches
|
from data import get_previous_time, set_current_time, check_new_database_matches
|
||||||
from interface import splash, log, ERR, INF, stdout, stderr
|
from interface import splash, log, ERR, INF, stdout, stderr
|
||||||
from module import Match, Metric, Pit
|
from module import Match, Metric, Pit
|
||||||
@ -171,10 +170,6 @@ config_path = "config.json"
|
|||||||
def main(send, verbose = False, profile = False, debug = False):
|
def main(send, verbose = False, profile = False, debug = False):
|
||||||
|
|
||||||
def close_all():
|
def close_all():
|
||||||
if "exec_threads" in locals():
|
|
||||||
exec_threads.terminate()
|
|
||||||
exec_threads.join()
|
|
||||||
exec_threads.close()
|
|
||||||
if "client" in locals():
|
if "client" in locals():
|
||||||
client.close()
|
client.close()
|
||||||
if "f" in locals():
|
if "f" in locals():
|
||||||
@ -196,14 +191,11 @@ def main(send, verbose = False, profile = False, debug = False):
|
|||||||
|
|
||||||
send(stdout, INF, "current time: " + str(loop_start))
|
send(stdout, INF, "current time: " + str(loop_start))
|
||||||
|
|
||||||
config = {}
|
config = Configuration(config_path)
|
||||||
|
|
||||||
if load_config(config_path, config):
|
|
||||||
raise ConfigurationError("could not find config at <" + config_path + ">, generating blank config and exiting", 110)
|
|
||||||
|
|
||||||
send(stdout, INF, "found and loaded config at <" + config_path + ">")
|
send(stdout, INF, "found and loaded config at <" + config_path + ">")
|
||||||
|
|
||||||
apikey, tbakey, preference, sync = parse_config_persistent(send, config)
|
apikey, tbakey = config.database, config.tba
|
||||||
|
|
||||||
send(stdout, INF, "found and loaded database and tba keys")
|
send(stdout, INF, "found and loaded database and tba keys")
|
||||||
|
|
||||||
@ -213,13 +205,10 @@ def main(send, verbose = False, profile = False, debug = False):
|
|||||||
previous_time = get_previous_time(client)
|
previous_time = get_previous_time(client)
|
||||||
send(stdout, INF, "analysis backtimed to: " + str(previous_time))
|
send(stdout, INF, "analysis backtimed to: " + str(previous_time))
|
||||||
|
|
||||||
config = resolve_config_conflicts(send, client, config, preference, sync)
|
config.resolve_config_conflicts(send, client)
|
||||||
|
|
||||||
|
config_modules, competition = config.modules, config.competition
|
||||||
|
|
||||||
exec_threads, config_modules = parse_config_variable(send, config)
|
|
||||||
if 'competition' in config['variable']:
|
|
||||||
competition = config['variable']['competition']
|
|
||||||
else:
|
|
||||||
competition = pull.get_team_competition()
|
|
||||||
for m in config_modules:
|
for m in config_modules:
|
||||||
if m in modules:
|
if m in modules:
|
||||||
start = time.time()
|
start = time.time()
|
||||||
@ -227,7 +216,7 @@ def main(send, verbose = False, profile = False, debug = False):
|
|||||||
valid = current_module.validate_config()
|
valid = current_module.validate_config()
|
||||||
if not valid:
|
if not valid:
|
||||||
continue
|
continue
|
||||||
current_module.run(exec_threads)
|
current_module.run()
|
||||||
send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds")
|
send(stdout, INF, m + " module finished in " + str(time.time() - start) + " seconds")
|
||||||
if debug:
|
if debug:
|
||||||
f = open(m + ".log", "w+")
|
f = open(m + ".log", "w+")
|
||||||
@ -360,7 +349,6 @@ def restart(pid_path):
|
|||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
if sys.platform.startswith("win"):
|
if sys.platform.startswith("win"):
|
||||||
freeze_support()
|
|
||||||
start(None, verbose = True)
|
start(None, verbose = True)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
@ -1,27 +0,0 @@
|
|||||||
{
|
|
||||||
"persistent": {
|
|
||||||
"type": "dict",
|
|
||||||
"require_all": true,
|
|
||||||
"schema": {
|
|
||||||
"key": {
|
|
||||||
"type": "dict",
|
|
||||||
"require_all":true,
|
|
||||||
"schema": {
|
|
||||||
"database": {"type":"string"},
|
|
||||||
"tba": {"type": "string"},
|
|
||||||
"tra": {
|
|
||||||
"type": "dict",
|
|
||||||
"require_all": true,
|
|
||||||
"schema": {
|
|
||||||
"CLIENT_ID": {"type": "string"},
|
|
||||||
"CLIENT_SECRET": {"type": "string"},
|
|
||||||
"url": {"type": "string"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"config-preference": {"type": "string", "required": true},
|
|
||||||
"synchronize-config": {"type": "boolean", "required": true}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
Loading…
Reference in New Issue
Block a user